diff --git a/README.md b/README.md index 5f75374bee..3caf3bb47b 100644 --- a/README.md +++ b/README.md @@ -1039,3 +1039,9 @@ https://www.firecrawl.dev/app/settings?tab=data-sources. Never infer consent fro failed lookup or automatically retry an acceptance. The API remains authoritative for organization access and acceptance authority. After confirmed success, rerun the original provider command; its normal credits apply. + +### Alexandria receipts and retries + +Alexandria execution JSON includes an additive `receipt`: `creditsUsed` is actual reported usage (missing means unknown), `requestId` is the client idempotency identity, and `operationId`/`operationType` identify the server scrape. Existing response fields remain available. IDs, reported credits, and available retry delays print to stderr. + +Use the same request ID to recover pending or uncertain execution. Completed results, including failures, replay under the same ID; a deliberate new execution needs a new ID and may charge again. Never automatically rotate an uncertain ID. Structured failures preserve available status, code, action and retry metadata. diff --git a/package.json b/package.json index 82107a224c..3bc6f8e2de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.23.4-alexandria-beta.14", + "version": "1.23.4-alexandria-beta.15", "publishConfig": { "tag": "alexandria" }, diff --git a/src/__tests__/alexandria-beta.test.ts b/src/__tests__/alexandria-beta.test.ts index 28a009c7a2..83d7332bbf 100644 --- a/src/__tests__/alexandria-beta.test.ts +++ b/src/__tests__/alexandria-beta.test.ts @@ -431,6 +431,12 @@ it('sends provider calls to Scrape with a stable retry ID and preserves the rece expect(JSON.parse(result.stdout)).toEqual({ ...response, requestId: 'retry-1', + receipt: { + creditsUsed: 1, + requestId: 'retry-1', + operationId: 'scrape-1', + operationType: 'scrape', + }, }); } expect(requests).toHaveLength(2); diff --git a/src/__tests__/utils/receipt.test.ts b/src/__tests__/utils/receipt.test.ts new file mode 100644 index 0000000000..9b8510b9f9 --- /dev/null +++ b/src/__tests__/utils/receipt.test.ts @@ -0,0 +1,60 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { apiFailure } from '../../commands/alexandria'; +import { receiptFor } from '../../utils/receipt'; + +afterEach(() => vi.useRealTimers()); + +it('keeps actual zero charges and separates client identity from the server operation', () => { + expect( + receiptFor( + { scrape_id: 'server-1', data: { creditsCost: 0 } }, + 'scrape', + 'client-1' + ) + ).toEqual({ + creditsUsed: 0, + requestId: 'client-1', + operationId: 'server-1', + operationType: 'scrape', + }); + expect(receiptFor({}, 'scrape')).toEqual({}); + expect( + receiptFor({ metadata: { creditsUsed: -1 } }, 'scrape') + ).not.toHaveProperty('creditsUsed'); +}); + +it('preserves actionable errors without serializing transport credentials', () => { + const failure = apiFailure({ + response: { + status: 429, + data: { + error: 'Limited', + code: 'RATE_LIMITED', + requestId: 'request-1', + retry_after_seconds: 1.5, + }, + config: { headers: { authorization: 'secret' } }, + }, + }); + expect(failure).toEqual({ + success: false, + error: 'Limited', + code: 'RATE_LIMITED', + requestId: 'request-1', + status: 429, + retryAfterSeconds: 2, + }); + expect(JSON.stringify(failure)).not.toContain('secret'); +}); + +it('handles HTTP-date retry delays without treating invalid numeric delays as dates', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-09-17T00:00:00Z')); + const failure = (retry: string) => + apiFailure({ + response: { status: 429, headers: { 'retry-after': retry } }, + }); + expect(failure('Thu, 17 Sep 2026 00:00:03 GMT').retryAfterSeconds).toBe(3); + for (const value of ['-5', 'nonsense', '']) + expect(failure(value)).not.toHaveProperty('retryAfterSeconds'); +}); diff --git a/src/commands/alexandria.ts b/src/commands/alexandria.ts index 4766dbd108..77f57caa96 100644 --- a/src/commands/alexandria.ts +++ b/src/commands/alexandria.ts @@ -1,6 +1,7 @@ +import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; import { randomUUID } from 'node:crypto'; import { Command, Option } from 'commander'; -import { SdkError, type AlexandriaCall } from 'firecrawl'; +import { type AlexandriaCall } from 'firecrawl'; import { getClient } from '../utils/client'; import { getApiKey } from '../utils/config'; import { writeOutput } from '../utils/output'; @@ -51,19 +52,41 @@ export function buildCalls(addresses: string[], values: string[] = []): Call[] { }); } -export function apiFailure(error: unknown): Record { +export function apiFailure(error: unknown): Record { + const candidate = error as any; const body = - (error as any)?.response?.data ?? - (error instanceof SdkError - ? { - error: error.message, - code: error.code, - chargeId: error.chargeId, - requiresAction: - (error.details as any)?.requiresAction ?? error.requiresAction, - } - : undefined); - return { + candidate?.response?.data ?? candidate?.details ?? candidate ?? {}; + const headers = candidate?.response?.headers; + const retryHeader = + typeof headers?.get === 'function' + ? headers.get('retry-after') + : headers?.['retry-after']; + const rawRetry = + body?.retry_after_seconds ?? + body?.retryAfterSeconds ?? + body?.details?.retryAfterSeconds ?? + candidate?.retryAfterSeconds ?? + retryHeader; + let retryAfterSeconds: number | undefined; + if ( + typeof rawRetry === 'number' || + (typeof rawRetry === 'string' && /^\d+(\.\d+)?$/.test(rawRetry)) + ) { + const seconds = Number(rawRetry); + if (Number.isFinite(seconds) && seconds >= 0) + retryAfterSeconds = Math.ceil(seconds); + } else if ( + typeof retryHeader === 'string' && + /^[A-Za-z]{3}, /.test(retryHeader) + ) { + const timestamp = Date.parse(retryHeader); + if (Number.isFinite(timestamp)) + retryAfterSeconds = Math.max( + 0, + Math.ceil((timestamp - Date.now()) / 1000) + ); + } + const result: Record = { success: false, error: typeof body?.error === 'string' @@ -71,10 +94,26 @@ export function apiFailure(error: unknown): Record { : error instanceof Error ? error.message : 'Request failed', - ...(typeof body?.code === 'string' && { code: body.code }), - ...(typeof body?.chargeId === 'string' && { chargeId: body.chargeId }), - ...(body?.requiresAction && { requiresAction: body.requiresAction }), }; + for (const key of [ + 'code', + 'chargeId', + 'requestId', + 'scrapeId', + 'scrape_id', + 'id', + ]) { + const value = body?.[key] ?? candidate?.[key]; + if (typeof value === 'string') result[key] = value; + } + const status = + candidate?.response?.status ?? candidate?.status ?? body?.status; + if (typeof status === 'number') result.status = status; + const action = body?.requiresAction ?? candidate?.requiresAction; + if (action) result.requiresAction = action; + if (retryAfterSeconds !== undefined) + result.retryAfterSeconds = retryAfterSeconds; + return result; } export async function requestAlexandria( @@ -107,7 +146,10 @@ export async function requestAlexandria( } catch (error) { envelope = apiFailure(error); } - return { ...envelope, requestId }; + const receipt = receiptFor(envelope, 'scrape', requestId); + printReceipt(receipt, false); + printRetry(envelope); + return { ...envelope, requestId, receipt }; } export async function handleAlexandria( @@ -162,7 +204,10 @@ export function createFindToolsCommand(): Command { '--request ', 'A complete next request returned by Find Tools' ) - .option('--request-id ', 'Reuse for an identical retry') + .option( + '--request-id ', + 'Reuse to recover the same execution; completed results replay' + ) .option('-k, --api-key ', 'Firecrawl API key') .option('--api-url ', 'Firecrawl API URL') .option('-o, --output ', 'Output file') @@ -208,7 +253,7 @@ export function addAlexandriaScrapeOptions(command: Command): void { .addOption( new Option( '--request-id ', - 'Reuse the same ID only for an identical tool retry' + 'Recover the same execution; completed results replay. Never replace an uncertain ID automatically.' ) ) .addOption( diff --git a/src/utils/receipt.ts b/src/utils/receipt.ts new file mode 100644 index 0000000000..8be60ddf5a --- /dev/null +++ b/src/utils/receipt.ts @@ -0,0 +1,48 @@ +export interface Receipt { + creditsUsed?: number; + requestId?: string; + operationId?: string; + operationType?: 'scrape' | 'search'; +} + +export function receiptFor( + value: any, + operationType: 'scrape' | 'search', + requestId?: string +): Receipt { + const credits = + operationType === 'search' + ? value?.creditsUsed + : (value?.metadata?.creditsUsed ?? + value?.creditsCost ?? + value?.data?.creditsCost); + const operationId = + operationType === 'search' + ? value?.id + : (value?.metadata?.scrapeId ?? value?.scrape_id ?? value?.scrapeId); + return { + ...(typeof credits === 'number' && Number.isFinite(credits) && credits >= 0 + ? { creditsUsed: credits } + : {}), + ...(requestId ? { requestId } : {}), + ...(typeof operationId === 'string' && operationId + ? { operationId, operationType } + : {}), + }; +} + +export function printReceipt(receipt: Receipt, includeRequestId = true): void { + if (includeRequestId && receipt.requestId) + console.error(`Request ID: ${receipt.requestId}`); + if (receipt.operationId) + console.error( + `${receipt.operationType === 'search' ? 'Search' : 'Scrape'} ID: ${receipt.operationId}` + ); + if (receipt.creditsUsed !== undefined) + console.error(`Credits: ${receipt.creditsUsed}`); +} + +export function printRetry(failure: Record): void { + if (typeof failure.retryAfterSeconds === 'number') + console.error(`Retry after: ${failure.retryAfterSeconds}s`); +}