diff --git a/README.md b/README.md index 5f75374bee..f1db0f1f56 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ When using a custom API URL (anything other than `https://api.firecrawl.dev`), a ### `scrape` - Scrape URLs -Extract content from any webpage. Pass multiple URLs to scrape them concurrently -- each result is saved to `.firecrawl/` automatically. +Extract content from any webpage. A single URL writes to stdout unless `-o` is supplied. Pass multiple URLs to scrape them concurrently; without output flags, successful results are saved to `.firecrawl/`, while failed URLs are reported on stderr without per-URL files. With `--json` or `-o `, results form one JSON array in input order, written to stdout or the requested file. Each item contains `url`, `success`, and the full `data` (including metadata) or an `error`. `--pretty` indents the array. Any failed URL makes the command exit nonzero, after successful results and errors have been saved. ```bash # Basic usage (outputs markdown) @@ -234,29 +234,30 @@ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.f #### Scrape Options -| Option | Description | -| -------------------------- | ------------------------------------------------------- | -| `-f, --format ` | Output format(s), comma-separated | -| `-H, --html` | Shortcut for `--format html` | -| `-S, --summary` | Shortcut for `--format summary` | -| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | -| `--wait-for ` | Wait time before scraping (for JS-rendered content) | -| `--screenshot` | Take a screenshot | -| `--full-page-screenshot` | Take a full page screenshot | -| `--include-tags ` | Only include specific HTML tags | -| `--exclude-tags ` | Exclude specific HTML tags | -| `--max-age ` | Maximum age of cached content in milliseconds | -| `--lockdown` | Enable lockdown mode for the scrape | -| `--redact-pii` | Redact personally identifiable information from output | -| `--schema ` | JSON schema for structured extraction | -| `--schema-file ` | Path to JSON schema file for structured extraction | -| `--actions ` | JSON actions array to run during scrape | -| `--actions-file ` | Path to JSON actions file | -| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | -| `-o, --output ` | Save output to file | -| `--json` | Output as JSON format | -| `--pretty` | Pretty print JSON output | -| `--timing` | Show request timing info | +| Option | Description | +| -------------------------- | ------------------------------------------------------------ | +| `-f, --format ` | Output format(s), comma-separated | +| `-H, --html` | Shortcut for `--format html` | +| `-S, --summary` | Shortcut for `--format summary` | +| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | +| `--wait-for ` | Wait time before scraping (for JS-rendered content) | +| `--timeout ` | Request timeout in milliseconds | +| `--screenshot` | Take a screenshot | +| `--full-page-screenshot` | Take a full page screenshot | +| `--include-tags ` | Only include specific HTML tags | +| `--exclude-tags ` | Exclude specific HTML tags | +| `--max-age ` | Maximum cached-content age; use `0` to request fresh content | +| `--lockdown` | Enable lockdown mode for the scrape | +| `--redact-pii` | Redact personally identifiable information from output | +| `--schema ` | JSON schema for structured extraction | +| `--schema-file ` | Path to JSON schema file for structured extraction | +| `--actions ` | JSON actions array to run during scrape | +| `--actions-file ` | Path to JSON actions file | +| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | +| `--pretty` | Pretty print JSON output | +| `--timing` | Show request timing info | #### Available Formats @@ -345,6 +346,8 @@ firecrawl search "API documentation" --scrape --scrape-formats markdown,links firecrawl search "AI data tools" ``` +Search prints the returned Search ID and credit usage to stderr when available, keeping stdout suitable for piping. `--json` and `--pretty` preserve the response metadata even when no results match; `-o` also saves empty results. + #### Search Options | Option | Description | @@ -364,6 +367,7 @@ firecrawl search "AI data tools" | `--only-main-content` | Include only main content when scraping (default: true) | | `-o, --output ` | Save to file | | `--json` | Output as compact JSON | +| `--pretty` | Output as pretty-printed JSON | #### Examples @@ -914,6 +918,22 @@ firecrawl https://example.com -o output.md firecrawl https://example.com --format links --pretty ``` +### Receipts, failures, and retries + +Search and URL scrape JSON output add a root `receipt` while preserving existing response fields; multi-URL JSON has a `receipt` on each result item. Raw text output stays unchanged. Available receipt fields are: + +- `creditsUsed`: actual credits reported by the response, including zero; absent means unknown, not free. A tool's catalog price is not a charge receipt. +- `requestId`: client idempotency ID for Alexandria calls. +- `operationId` and `operationType` (`search` or `scrape`): the returned server operation ID and its kind, for tracing the operation. + +These identifiers serve different purposes. `--request-id` controls Alexandria retry identity; it is not supported for ordinary URL scrape. Diagnostics print available IDs, credits, and retry timing to stderr. Keep stderr separate from JSON stdout; `2>&1` combines them and is not parseable JSON. + +With `--json` or `-o`, failed search/scrape calls write structured error output before exiting nonzero. A single scrape failure writes JSON even when the requested filename ends in `.md`. Multi-URL explicit output preserves successful results and failures in input order. Inspect the exit code and error fields before treating a saved file as usable content. Missing receipt fields do not establish whether a timed-out operation was billed. + +On a rate limit, wait at least the returned retry delay when available; otherwise use bounded exponential backoff. API keys on the same team share limits, which vary by plan and endpoint. See the Alexandria retry guidance below for provider executions. + +For URL scrape, `--timeout ` sets the server-side scrape timeout (the SDK allows transport overhead) and `--max-age 0` requests fresh content. Freshness does not guarantee the source returns a successful page, and a timeout is not proof that upstream work stopped. Inspect returned status/error metadata as well as content. + ### Format Behavior - **Single format**: Outputs raw content (markdown text, HTML, etc.) @@ -1039,3 +1059,9 @@ https://www.firecrawl.dev/app/settings?tab=data-sources. Never infer consent fro failed lookup or automatically retry an acceptance. The API remains authoritative for organization access and acceptance authority. After confirmed success, rerun the original provider command; its normal credits apply. + +### Alexandria receipts and retries + +Alexandria execution JSON includes an additive `receipt`: `creditsUsed` is actual reported usage (missing means unknown), `requestId` is the client idempotency identity, and `operationId`/`operationType` identify the server scrape. Existing response fields remain available. IDs, reported credits, and available retry delays print to stderr. + +Use the same request ID to recover pending or uncertain execution. Completed results, including failures, replay under the same ID; a deliberate new execution needs a new ID and may charge again. Never automatically rotate an uncertain ID. Structured failures preserve available status, code, action and retry metadata. diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 16cfa8a03a..3f92247e36 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -23,7 +23,7 @@ firecrawl scrape "" --only-main-content -o .firecrawl/page.md # Wait for JS to render, then scrape firecrawl scrape "" --wait-for 3000 -o .firecrawl/page.md -# Multiple URLs (markdown only; each saved to .firecrawl/; -o is ignored) +# Multiple URLs (successful results saved to .firecrawl/; failures reported) firecrawl scrape https://example.com https://example.com/blog https://example.com/docs # Get markdown and links together @@ -47,11 +47,22 @@ firecrawl scrape "https://example.com/report.pdf" --max-pages 5 --json -o .firec The cap applies to each PDF, not the whole command or total credits. Extra formats and options can add charges. The CLI does not quote page counts or costs before execution. Use JSON output to inspect the returned `metadata.numPages` (parsed), `metadata.totalPages` (document total), and `metadata.creditsUsed` when present; a smaller parsed count means the result is partial. +## Receipts and recovery + +Use `--json` to preserve metadata and the additive `receipt` (at the root for a single scrape, on each result item for multiple URLs). `receipt.creditsUsed` is actual returned usage, including zero; missing means unknown. Existing `metadata.creditsUsed` remains available when returned. `receipt.operationId` identifies the server scrape; Alexandria `receipt.requestId` is a separate client idempotency ID. Available IDs, credit usage, and retry timing print to stderr. Keep stderr separate from JSON stdout. + +Failures with `--json` or `-o` write structured errors before exiting nonzero, even if the filename ends in `.md`. Inspect the exit code and saved error/status fields before using the content. A successful transport response can still contain a refused or unsuccessful page; do not treat it as task completion or infer a refund. + +- Use `--timeout ` to set the server-side scrape timeout; the SDK allows transport overhead. A timeout does not prove the operation stopped or cost zero credits. +- Use `--max-age 0` when fresh URL content is required. This does not guarantee the source page succeeds. +- On rate limits, honor the returned retry delay when available, otherwise use bounded exponential backoff. Limits are shared across a team's keys and depend on plan and endpoint. +- For Alexandria, keep the same `--request-id` while an operation is unresolved. A completed failure can replay under the same ID; starting a new attempt requires a new ID and may charge again. Never rotate IDs automatically. Ordinary URL scrape does not support `--request-id`. + ## Tips - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). - **Scrape handles static pages and JS-rendered SPAs.** Escalate to `interact` when the page needs interaction (clicks, form fills, pagination) or scrape misses content. -- Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. This mode saves markdown only and ignores `-o`; other requested formats are dropped. If markdown wasn't requested, the whole JSON response is written into the `.md` file. +- Multiple URLs are scraped concurrently. Use `--json` for an ordered JSON array on stdout or `-o results.json` to save it. Each item contains `url`, `success`, and full `data` with metadata or an `error`; any failed URL makes the command exit nonzero. Without either flag, successful results are saved under `.firecrawl/` as markdown when available, otherwise JSON in a `.md` file. Failed URLs are reported on stderr without creating per-URL files. Check `firecrawl --status` for your concurrency limit. - Single format outputs raw content. Multiple formats (e.g., `--format markdown,links`) output JSON. - Always quote URLs — shell interprets `?` and `&` as special characters. - Naming convention: `.firecrawl/{site}-{path}.md` diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 7831635fac..cf3598817d 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -28,7 +28,7 @@ Run `firecrawl search --help` for the full option list. `--categories developer` weighs the developer index beside ordinary web results in this same call (no passage control, no index filters). `--categories research` is a website filter, not the paper index. Dedicated skills: [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) and [firecrawl-research-index](../firecrawl-research-index/SKILL.md). -**Done when:** results are saved under `.firecrawl/`, verified non-empty, processed for the request, and one feedback event is sent within the time window (unless opted out). +**Done when:** the response is saved under `.firecrawl/`, results or an empty result set have been inspected and handled for the request, and eligible feedback is sent within the time window (unless opted out). ## Alexandria in normal search @@ -40,6 +40,14 @@ Use `find-tools` only for an explicitly requested tool set or a missing contract If no returned tool covers the country/market/segment or required inputs, continue with ordinary web results. Do not exhaust the catalogue or pay for adjacent tools just to probe coverage. `--sources web` explicitly opts out of Alexandria; `--sources web --domain-tools` retains domain matches only. +## Receipts and failures + +JSON output preserves the response and additive `receipt`, including empty results. Read `receipt.creditsUsed` for actual reported usage (zero is valid; missing means unknown) and `receipt.operationId` with `operationType: "search"` for the server search ID. Existing `id` and `creditsUsed` fields remain available. Search IDs identify results; they are not Alexandria client idempotency IDs. + +Available IDs, credits, and retry timing print to stderr. Keep stderr separate from JSON stdout. Failed calls with `--json` or `-o` write structured errors before exiting nonzero; check the exit code and `success` before using the file. Empty successful searches still write the requested output and retain their metadata. + +On rate limits, wait at least the returned retry delay when available; otherwise use bounded exponential backoff. API keys on one team share limits, which vary by plan and endpoint. Do not invent a universal requests-per-minute quota or infer zero billing from a missing receipt. + ## Tips - **`--highlights` on by default:** results are query-relevant excerpts, not full-page snippets. Use `--no-highlights` for the original snippets. @@ -66,13 +74,13 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Idempotent:** re-submitting for the same search id returns success but no extra refund. - **`--silent &`** is the right pattern — exit code 0 even on failure, so a rejected/expired call never crashes your pipeline. -Verify the search returned results before reading its `id`. Zero-result searches write no output file, so the file may be missing — or left over from an earlier search. The guard below skips feedback when the file is missing or has zero results; call `search-feedback` only inside it: +Verify the search succeeded before reading its `id`. Empty successful searches preserve their JSON output; failed searches contain error output. The guard below only sends feedback for a successful response with an ID and nonempty results; call `search-feedback` only inside it: ```bash # Send once per search. Rate honestly and replace the placeholder with the # rating that matches what actually happened. The two fields shown # satisfy the substantive-content rule for every rating. -if SEARCH_ID=$(jq -er 'select(any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then +if SEARCH_ID=$(jq -er 'select(.success == true and any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then firecrawl search-feedback "$SEARCH_ID" \ --rating "" \ --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ diff --git a/src/__tests__/alexandria-beta.test.ts b/src/__tests__/alexandria-beta.test.ts index 28a009c7a2..211b58f8ae 100644 --- a/src/__tests__/alexandria-beta.test.ts +++ b/src/__tests__/alexandria-beta.test.ts @@ -400,7 +400,14 @@ it('preserves mixed search results, tools and billing metadata', async () => { }; const result = await cli(['search', 'pizza hut', '--json']); expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toEqual(response); + expect(JSON.parse(result.stdout)).toEqual({ + ...response, + receipt: { + creditsUsed: 2, + operationId: 'search-1', + operationType: 'search', + }, + }); expect(requests[0]).toMatchObject({ url: '/v2/search', headers: { authorization: 'Bearer fc-test' }, @@ -431,6 +438,12 @@ it('sends provider calls to Scrape with a stable retry ID and preserves the rece expect(JSON.parse(result.stdout)).toEqual({ ...response, requestId: 'retry-1', + receipt: { + creditsUsed: 1, + requestId: 'retry-1', + operationId: 'scrape-1', + operationType: 'scrape', + }, }); } expect(requests).toHaveLength(2); @@ -535,7 +548,7 @@ it('keeps URL scrape tool contracts in the output', async () => { }; const result = await cli(['scrape', 'https://example.com', '--domain-tools']); expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toEqual(response.data); + expect(JSON.parse(result.stdout)).toEqual({ ...response.data, receipt: {} }); expect(requests[0]).toMatchObject({ url: '/v2/scrape', body: { url: 'https://example.com', domainTools: true }, @@ -898,3 +911,97 @@ it('fails clearly on an unknown thread', async () => { expect(malformed.code).toBe(1); expect(requests).toHaveLength(1); }); + +it('writes a structured plain-scrape refusal to the requested file before exiting', async () => { + status = 403; + response = { + success: false, + error: 'Access denied', + code: 'ACCESS_DENIED', + requestId: 'server-error-1', + }; + const output = join(home, 'refusal.md'); + const result = await cli([ + 'scrape', + 'https://example.com', + '--json', + '-o', + output, + ]); + expect(result.code).toBe(1); + expect(result.stdout).toBe(''); + expect(JSON.parse(readFileSync(output, 'utf8'))).toMatchObject({ + success: false, + error: 'Access denied', + code: 'ACCESS_DENIED', + status: 403, + requestId: 'server-error-1', + }); + expect(requests).toHaveLength(1); +}); + +it('keeps raw scrape stdout pipeable and reports returned cache and charges', async () => { + response = { + success: true, + data: { + markdown: 'fixture', + metadata: { + scrapeId: 'plain-1', + creditsUsed: 0, + cacheState: 'hit', + cachedAt: '2026-09-17T00:00:00Z', + }, + }, + }; + const result = await cli([ + 'scrape', + 'https://example.com', + '--timeout', + '2500', + ]); + expect(result.code).toBe(0); + expect(result.stdout).toBe('fixture\n'); + expect(result.stderr).toContain('Scrape ID: plain-1'); + expect(result.stderr).toContain('Credits: 0'); + expect(result.stderr).toContain('Cache: hit'); + expect(result.stderr).toContain('Cached at: 2026-09-17T00:00:00Z'); + expect(requests[0].body.timeout).toBe(2500); + expect(requests[0].body).not.toHaveProperty('autoResume'); + const invalid = await cli([ + 'scrape', + 'https://example.com', + '--timeout', + '0', + ]); + expect(invalid.code).toBe(1); + expect(invalid.stderr).toContain('positive integer in milliseconds'); + expect(requests).toHaveLength(1); +}); + +it('accepts search --pretty and preserves empty results and receipts', async () => { + response = { + success: true, + id: 'pretty-search', + creditsUsed: 0, + data: { web: [] }, + }; + const result = await cli([ + 'search', + 'fixture', + '--sources', + 'web', + '--pretty', + ]); + expect(result.code).toBe(0); + expect(result.stdout).toContain('\n "success": true'); + expect(JSON.parse(result.stdout)).toMatchObject({ + ...response, + receipt: { + creditsUsed: 0, + operationId: 'pretty-search', + operationType: 'search', + }, + }); + expect(result.stderr).toContain('Search ID: pretty-search'); + expect(requests).toHaveLength(1); +}); diff --git a/src/__tests__/commands/multi-scrape.test.ts b/src/__tests__/commands/multi-scrape.test.ts new file mode 100644 index 0000000000..5389dd9b44 --- /dev/null +++ b/src/__tests__/commands/multi-scrape.test.ts @@ -0,0 +1,157 @@ +import * as fs from 'fs'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { handleMultiScrapeCommand } from '../../commands/scrape'; +import { getClient } from '../../utils/client'; +import { clearInteractSession } from '../../utils/interact-session'; + +vi.mock('../../utils/client', () => ({ + getClient: vi.fn(), + isKeylessMode: () => false, + keylessRequest: vi.fn(), +})); +vi.mock('../../utils/interact-session', () => ({ + saveInteractSession: vi.fn(), + clearInteractSession: vi.fn(), +})); +vi.mock('fs', async () => ({ + ...(await vi.importActual('fs')), + existsSync: vi.fn(() => false), + mkdirSync: vi.fn(), + writeFileSync: vi.fn(), +})); + +const urls = ['https://example.com/a', 'https://example.com/b']; +const documents = urls.map((url, index) => ({ + markdown: `Page ${index}`, + html: `

Page ${index}

`, + metadata: { sourceURL: url, creditsUsed: index + 1, scrapeId: `id-${index}` }, +})); + +describe('multi-URL scrape output', () => { + const scrape = vi.fn(); + let originalExitCode: typeof process.exitCode; + + beforeEach(() => { + originalExitCode = process.exitCode; + process.exitCode = undefined; + vi.clearAllMocks(); + scrape.mockReset(); + vi.mocked(getClient).mockReturnValue({ scrape } as any); + vi.spyOn(process.stdout, 'write').mockReturnValue(true); + vi.spyOn(process.stderr, 'write').mockReturnValue(true); + }); + + afterEach(() => { + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + }); + + it('writes one ordered collection to the requested path despite out-of-order completion', async () => { + let resolveFirst!: (value: unknown) => void; + scrape.mockImplementation(async (url) => { + if (url === urls[0]) + return new Promise((resolve) => { + resolveFirst = resolve; + }); + setImmediate(() => resolveFirst(documents[0])); + return documents[1]; + }); + await handleMultiScrapeCommand(urls, { + url: urls[0], + output: 'out/results.txt', + pretty: true, + }); + expect(fs.writeFileSync).toHaveBeenCalledTimes(1); + const [path, content] = vi.mocked(fs.writeFileSync).mock.calls[0]; + expect(path).toBe('out/results.txt'); + expect(content).toBe( + JSON.stringify( + documents.map((data, index) => ({ + url: urls[index], + success: true, + data, + receipt: { + creditsUsed: index + 1, + operationId: `id-${index}`, + operationType: 'scrape', + }, + })), + null, + 2 + ) + ); + expect(fs.mkdirSync).not.toHaveBeenCalledWith( + '.firecrawl', + expect.anything() + ); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(clearInteractSession).toHaveBeenCalledOnce(); + expect(process.exitCode).toBeUndefined(); + }); + + it('prints valid JSON with full metadata on stdout without per-URL files', async () => { + scrape + .mockResolvedValueOnce(documents[0]) + .mockResolvedValueOnce(documents[1]); + await handleMultiScrapeCommand(urls, { url: urls[0], json: true }); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + const output = JSON.parse( + String(vi.mocked(process.stdout.write).mock.calls[0][0]) + ); + expect(output.map((item: any) => item.data)).toEqual(documents); + expect(fs.writeFileSync).not.toHaveBeenCalled(); + expect(fs.mkdirSync).not.toHaveBeenCalled(); + }); + + it.each([false, true])( + 'preserves errors and successful results with failure exit status (all failed: %s)', + async (allFailed) => { + if (allFailed) scrape.mockRejectedValueOnce(new Error('First failed')); + else scrape.mockResolvedValueOnce(documents[0]); + scrape.mockRejectedValueOnce(new Error('Second failed')); + await handleMultiScrapeCommand(urls, { + url: urls[0], + json: true, + output: 'results.json', + }); + const output = JSON.parse( + String(vi.mocked(fs.writeFileSync).mock.calls[0][1]) + ); + expect(output).toEqual([ + allFailed + ? { url: urls[0], success: false, error: 'First failed', receipt: {} } + : { + url: urls[0], + success: true, + data: documents[0], + receipt: { + operationId: 'id-0', + operationType: 'scrape', + creditsUsed: 1, + }, + }, + { url: urls[1], success: false, error: 'Second failed', receipt: {} }, + ]); + expect(process.exitCode).toBe(1); + expect(process.stdout.write).not.toHaveBeenCalled(); + } + ); + + it('keeps default per-file behavior and reports partial failure', async () => { + scrape + .mockResolvedValueOnce(documents[0]) + .mockRejectedValueOnce(new Error('Second failed')); + await handleMultiScrapeCommand(urls, { url: urls[0] }); + expect(fs.mkdirSync).toHaveBeenCalledWith('.firecrawl', { + recursive: true, + }); + expect(fs.writeFileSync).toHaveBeenCalledWith( + '.firecrawl/example.com-a.md', + 'Page 0', + 'utf-8' + ); + expect(fs.writeFileSync).toHaveBeenCalledTimes(1); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); +}); diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index b9c9606882..49571d85ac 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -432,6 +432,7 @@ describe('executeScrape', () => { expect(result).toEqual({ success: true, data: mockResponse, + receipt: {}, }); }); @@ -466,6 +467,7 @@ describe('executeScrape', () => { expect(result).toEqual({ success: false, error: errorMessage, + receipt: {}, }); }); @@ -477,7 +479,7 @@ describe('executeScrape', () => { }); expect(result.success).toBe(false); - expect(result.error).toBe('Unknown error occurred'); + expect(result.error).toBe('Request failed'); }); }); diff --git a/src/__tests__/commands/search.test.ts b/src/__tests__/commands/search.test.ts index aeded653fc..9a2986f5fb 100644 --- a/src/__tests__/commands/search.test.ts +++ b/src/__tests__/commands/search.test.ts @@ -637,6 +637,7 @@ describe('executeSearch', () => { expect(result).toEqual({ success: false, error: errorMessage, + receipt: {}, }); }); @@ -648,7 +649,7 @@ describe('executeSearch', () => { }); expect(result.success).toBe(false); - expect(result.error).toBe('Unknown error occurred'); + expect(result.error).toBe('Request failed'); }); }); diff --git a/src/__tests__/utils/output.test.ts b/src/__tests__/utils/output.test.ts index 8667768ece..795b9af732 100644 --- a/src/__tests__/utils/output.test.ts +++ b/src/__tests__/utils/output.test.ts @@ -107,7 +107,9 @@ describe('Output Utilities', () => { handleScrapeOutput({ success: false, error: 'API Error' }, ['markdown']); expect(consoleErrorSpy).toHaveBeenCalledWith('Error:', 'API Error'); - expect(processExitSpy).toHaveBeenCalledWith(1); + expect(processExitSpy).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + process.exitCode = undefined; }); it('should output raw markdown for single markdown format', () => { diff --git a/src/__tests__/utils/receipt.test.ts b/src/__tests__/utils/receipt.test.ts new file mode 100644 index 0000000000..9b8510b9f9 --- /dev/null +++ b/src/__tests__/utils/receipt.test.ts @@ -0,0 +1,60 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { apiFailure } from '../../commands/alexandria'; +import { receiptFor } from '../../utils/receipt'; + +afterEach(() => vi.useRealTimers()); + +it('keeps actual zero charges and separates client identity from the server operation', () => { + expect( + receiptFor( + { scrape_id: 'server-1', data: { creditsCost: 0 } }, + 'scrape', + 'client-1' + ) + ).toEqual({ + creditsUsed: 0, + requestId: 'client-1', + operationId: 'server-1', + operationType: 'scrape', + }); + expect(receiptFor({}, 'scrape')).toEqual({}); + expect( + receiptFor({ metadata: { creditsUsed: -1 } }, 'scrape') + ).not.toHaveProperty('creditsUsed'); +}); + +it('preserves actionable errors without serializing transport credentials', () => { + const failure = apiFailure({ + response: { + status: 429, + data: { + error: 'Limited', + code: 'RATE_LIMITED', + requestId: 'request-1', + retry_after_seconds: 1.5, + }, + config: { headers: { authorization: 'secret' } }, + }, + }); + expect(failure).toEqual({ + success: false, + error: 'Limited', + code: 'RATE_LIMITED', + requestId: 'request-1', + status: 429, + retryAfterSeconds: 2, + }); + expect(JSON.stringify(failure)).not.toContain('secret'); +}); + +it('handles HTTP-date retry delays without treating invalid numeric delays as dates', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-09-17T00:00:00Z')); + const failure = (retry: string) => + apiFailure({ + response: { status: 429, headers: { 'retry-after': retry } }, + }); + expect(failure('Thu, 17 Sep 2026 00:00:03 GMT').retryAfterSeconds).toBe(3); + for (const value of ['-5', 'nonsense', '']) + expect(failure(value)).not.toHaveProperty('retryAfterSeconds'); +}); diff --git a/src/commands/alexandria.ts b/src/commands/alexandria.ts index 4766dbd108..77f57caa96 100644 --- a/src/commands/alexandria.ts +++ b/src/commands/alexandria.ts @@ -1,6 +1,7 @@ +import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; import { randomUUID } from 'node:crypto'; import { Command, Option } from 'commander'; -import { SdkError, type AlexandriaCall } from 'firecrawl'; +import { type AlexandriaCall } from 'firecrawl'; import { getClient } from '../utils/client'; import { getApiKey } from '../utils/config'; import { writeOutput } from '../utils/output'; @@ -51,19 +52,41 @@ export function buildCalls(addresses: string[], values: string[] = []): Call[] { }); } -export function apiFailure(error: unknown): Record { +export function apiFailure(error: unknown): Record { + const candidate = error as any; const body = - (error as any)?.response?.data ?? - (error instanceof SdkError - ? { - error: error.message, - code: error.code, - chargeId: error.chargeId, - requiresAction: - (error.details as any)?.requiresAction ?? error.requiresAction, - } - : undefined); - return { + candidate?.response?.data ?? candidate?.details ?? candidate ?? {}; + const headers = candidate?.response?.headers; + const retryHeader = + typeof headers?.get === 'function' + ? headers.get('retry-after') + : headers?.['retry-after']; + const rawRetry = + body?.retry_after_seconds ?? + body?.retryAfterSeconds ?? + body?.details?.retryAfterSeconds ?? + candidate?.retryAfterSeconds ?? + retryHeader; + let retryAfterSeconds: number | undefined; + if ( + typeof rawRetry === 'number' || + (typeof rawRetry === 'string' && /^\d+(\.\d+)?$/.test(rawRetry)) + ) { + const seconds = Number(rawRetry); + if (Number.isFinite(seconds) && seconds >= 0) + retryAfterSeconds = Math.ceil(seconds); + } else if ( + typeof retryHeader === 'string' && + /^[A-Za-z]{3}, /.test(retryHeader) + ) { + const timestamp = Date.parse(retryHeader); + if (Number.isFinite(timestamp)) + retryAfterSeconds = Math.max( + 0, + Math.ceil((timestamp - Date.now()) / 1000) + ); + } + const result: Record = { success: false, error: typeof body?.error === 'string' @@ -71,10 +94,26 @@ export function apiFailure(error: unknown): Record { : error instanceof Error ? error.message : 'Request failed', - ...(typeof body?.code === 'string' && { code: body.code }), - ...(typeof body?.chargeId === 'string' && { chargeId: body.chargeId }), - ...(body?.requiresAction && { requiresAction: body.requiresAction }), }; + for (const key of [ + 'code', + 'chargeId', + 'requestId', + 'scrapeId', + 'scrape_id', + 'id', + ]) { + const value = body?.[key] ?? candidate?.[key]; + if (typeof value === 'string') result[key] = value; + } + const status = + candidate?.response?.status ?? candidate?.status ?? body?.status; + if (typeof status === 'number') result.status = status; + const action = body?.requiresAction ?? candidate?.requiresAction; + if (action) result.requiresAction = action; + if (retryAfterSeconds !== undefined) + result.retryAfterSeconds = retryAfterSeconds; + return result; } export async function requestAlexandria( @@ -107,7 +146,10 @@ export async function requestAlexandria( } catch (error) { envelope = apiFailure(error); } - return { ...envelope, requestId }; + const receipt = receiptFor(envelope, 'scrape', requestId); + printReceipt(receipt, false); + printRetry(envelope); + return { ...envelope, requestId, receipt }; } export async function handleAlexandria( @@ -162,7 +204,10 @@ export function createFindToolsCommand(): Command { '--request ', 'A complete next request returned by Find Tools' ) - .option('--request-id ', 'Reuse for an identical retry') + .option( + '--request-id ', + 'Reuse to recover the same execution; completed results replay' + ) .option('-k, --api-key ', 'Firecrawl API key') .option('--api-url ', 'Firecrawl API URL') .option('-o, --output ', 'Output file') @@ -208,7 +253,7 @@ export function addAlexandriaScrapeOptions(command: Command): void { .addOption( new Option( '--request-id ', - 'Reuse the same ID only for an identical tool retry' + 'Recover the same execution; completed results replay. Never replace an uncertain ID automatically.' ) ) .addOption( diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 3dc5d1780b..ed91b1c0a0 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -1,3 +1,5 @@ +import { apiFailure } from './alexandria'; +import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; /** * Scrape command implementation */ @@ -115,6 +117,13 @@ export async function executeScrape( scrapeParams.maxAge = options.maxAge; } + if (options.timeout !== undefined) { + if (!Number.isSafeInteger(options.timeout) || options.timeout <= 0) + throw new Error('--timeout must be a positive integer in milliseconds.'); + scrapeParams.timeout = options.timeout; + scrapeParams.autoResume = false; + } + if (options.maxPages !== undefined) { scrapeParams.parsers = [{ type: 'pdf', maxPages: options.maxPages }]; } @@ -155,9 +164,10 @@ export async function executeScrape( if (isKeylessMode(options.apiKey, options.apiUrl)) { // Keyless free tier: header-less request. The API identifies the CLI via // the `integration: 'cli'` field already in scrapeParams. + const { autoResume: _autoResume, ...wireParams } = scrapeParams; const json = await keylessRequest('/v2/scrape', { url: options.url, - ...scrapeParams, + ...wireParams, }); result = json?.data ?? json; } else { @@ -170,9 +180,14 @@ export async function executeScrape( const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime); + const receipt = receiptFor(result, 'scrape'); + printReceipt(receipt); + if (typeof result?.metadata?.cacheState === 'string') + console.error(`Cache: ${result.metadata.cacheState}`); + if (typeof result?.metadata?.cachedAt === 'string') + console.error(`Cached at: ${result.metadata.cachedAt}`); const scrapeId = result?.metadata?.scrapeId; if (scrapeId) { - process.stderr.write(`Scrape ID: ${scrapeId}\n`); try { saveInteractSession({ scrapeId, @@ -190,15 +205,17 @@ export async function executeScrape( return { success: true, data: result, + receipt, }; } catch (error) { const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime, error); - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown error occurred', - }; + const failure = apiFailure(error); + const receipt = receiptFor(failure, 'scrape'); + printReceipt(receipt); + printRetry(failure); + return { ...failure, success: false, receipt }; } } @@ -211,7 +228,14 @@ export async function handleScrapeCommand( const result = await executeScrape(options); // Query mode: output answer directly - if (options.query && result.success && result.data?.answer) { + if ( + options.query && + !options.json && + !options.pretty && + !options.output?.endsWith('.json') && + result.success && + result.data?.answer + ) { writeOutput(result.data.answer, options.output, !!options.output); return; } @@ -254,8 +278,7 @@ function urlToFilename(url: string): string { } /** - * Handle scrape for multiple URLs. - * Each result is saved as a separate file in .firecrawl/ + * Explicit output produces an ordered JSON collection; otherwise save per URL. */ export async function handleMultiScrapeCommand( urls: string[], @@ -263,9 +286,9 @@ export async function handleMultiScrapeCommand( ): Promise { const fs = await import('fs'); const path = await import('path'); - + const structuredOutput = !!options.output || !!options.json; const dir = '.firecrawl'; - if (!fs.existsSync(dir)) { + if (!structuredOutput && !fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } @@ -276,9 +299,7 @@ export async function handleMultiScrapeCommand( process.stderr.write(`Scraping ${total} URLs...\n`); const promises = urls.map(async (url) => { - const scrapeOptions: ScrapeOptions = { ...options, url }; - const result = await executeScrape(scrapeOptions); - + const result = await executeScrape({ ...options, url }); const currentCount = ++completedCount; if (!result.success) { @@ -286,33 +307,41 @@ export async function handleMultiScrapeCommand( process.stderr.write( `[${currentCount}/${total}] Error: ${url} - ${result.error}\n` ); - return; + } else if (structuredOutput) { + process.stderr.write(`[${currentCount}/${total}] Scraped: ${url}\n`); + } else { + const filename = urlToFilename(url); + const filepath = path.join(dir, filename); + const content = result.data?.markdown || JSON.stringify(result.data); + fs.writeFileSync(filepath, content, 'utf-8'); + process.stderr.write(`[${currentCount}/${total}] Saved: ${filepath}\n`); } - const filename = urlToFilename(url); - const filepath = path.join(dir, filename); - const content = result.data?.markdown || JSON.stringify(result.data); - fs.writeFileSync(filepath, content, 'utf-8'); - - process.stderr.write(`[${currentCount}/${total}] Saved: ${filepath}\n`); + // Avoid retaining every document in memory for the default per-file mode. + return structuredOutput ? { url, ...result } : undefined; }); - await Promise.all(promises); - + const results = await Promise.all(promises); clearInteractSession(); + + if (structuredOutput) { + writeOutput( + JSON.stringify(results, null, options.pretty ? 2 : undefined), + options.output, + !!options.output + ); + } + process.stderr.write( `\nCompleted: ${completedCount - errorCount}/${total} succeeded` ); if (errorCount > 0) { process.stderr.write(`, ${errorCount} failed`); + process.exitCode = 1; } process.stderr.write( '\nTip: Use --scrape-id with interact to target a specific scrape.\n' ); - - if (errorCount === total) { - process.exit(1); - } } /** diff --git a/src/commands/search.ts b/src/commands/search.ts index 2bac898562..dd2fd1b15e 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,3 +1,4 @@ +import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; /** * Search command implementation */ @@ -141,16 +142,14 @@ export async function executeSearch( warning: envelope.warning, id: envelope.id, creditsUsed: envelope.creditsUsed, + receipt: receiptFor(envelope, 'search'), }; } catch (error) { + const failure = apiFailure(error); return { + ...failure, success: false, - error: - options.domainTools || options.sources?.includes('alexandria') - ? JSON.stringify(apiFailure(error)) - : error instanceof Error - ? error.message - : 'Unknown error occurred', + receipt: receiptFor(failure, 'search'), }; } } @@ -319,9 +318,18 @@ export async function handleSearchCommand( ): Promise { const result = await executeSearch(options); + printReceipt(result.receipt ?? {}); + printRetry(result as unknown as Record); if (!result.success) { console.error('Error:', result.error); - process.exit(1); + if (options.json || options.pretty || options.output) + writeOutput( + JSON.stringify(result, null, options.pretty ? 2 : undefined), + options.output, + !!options.output + ); + process.exitCode = 1; + return; } if (!result.data) { @@ -336,19 +344,19 @@ export async function handleSearchCommand( (result.data.news && result.data.news.length > 0) || (result.data.developer && result.data.developer.length > 0); - if (!hasResults && !(result.data.tools && (options.json || options.pretty))) { - console.log('No results found.'); - return; - } - let outputContent: string; // Use JSON format if --json or --pretty flag is set // --pretty implies JSON output - if (options.json || options.pretty) { + if ( + options.json || + options.pretty || + options.output?.toLowerCase().endsWith('.json') + ) { const jsonOutput: Record = { success: true, data: result.data, + receipt: result.receipt, }; if (result.warning) { @@ -366,7 +374,9 @@ export async function handleSearchCommand( : JSON.stringify(jsonOutput); } else { // Default to human-readable format - outputContent = formatSearchReadable(result.data, options); + outputContent = hasResults + ? formatSearchReadable(result.data, options) + : 'No results found.'; } writeOutput(outputContent, options.output, !!options.output); diff --git a/src/index.ts b/src/index.ts index ecb1ba8244..a93d7cdd17 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,7 +71,11 @@ import { handleEnvPullCommand } from './commands/env'; import { handleStatusCommand } from './commands/status'; import { handleDoctorCommand } from './commands/doctor'; import { isUrl, normalizeUrl } from './utils/url'; -import { parseMaxPages, parseScrapeOptions } from './utils/options'; +import { + parseMaxPages, + parseScrapeOptions, + parseScrapeTimeout, +} from './utils/options'; import { isJobId } from './utils/job'; import { ensureAuthenticated, printBanner } from './utils/auth'; import { maybeShowUpdateNotice } from './utils/update-notice'; @@ -356,7 +360,7 @@ program function createScrapeCommand(): Command { const scrapeCmd = new Command('scrape') .description( - 'Scrape one or more URLs. Multiple URLs are scraped concurrently and saved to .firecrawl/' + 'Scrape one or more URLs. Multiple URLs save to .firecrawl/ by default; --json or -o produces an ordered JSON array.' ) .argument('[urls...]', 'URL(s) to scrape') .option( @@ -373,6 +377,11 @@ function createScrapeCommand(): Command { 'Maximum PDF pages to parse (1-10000). PDFs cost 1 credit per parsed page; extra options may cost more.', parseMaxPages ) + .option( + '--timeout ', + 'Server-side scrape timeout in milliseconds', + parseScrapeTimeout + ) .option('--only-main-content', 'Include only main content', false) .option( '--wait-for ', @@ -1000,12 +1009,8 @@ function createSearchCommand(): Command { ) .option('--api-url ', 'API URL (overrides global --api-url)') .option('-o, --output ', 'Output file path (default: stdout)') - // .option( - // '-p, --pretty', - // 'Output as pretty JSON (default: human-readable)', - // false - // ) .option('--json', 'Output as compact JSON', false) + .option('--pretty', 'Output as pretty-printed JSON', false) .action(async (query, options) => { // Parse sources let sources: SearchSource[] = ['web', 'alexandria']; diff --git a/src/types/scrape.ts b/src/types/scrape.ts index 8f5969163d..591eaf25aa 100644 --- a/src/types/scrape.ts +++ b/src/types/scrape.ts @@ -1,3 +1,4 @@ +import type { Receipt } from '../utils/receipt'; /** * Types and interfaces for the scrape command */ @@ -32,6 +33,8 @@ export interface ScrapeOptions { onlyMainContent?: boolean; /** Wait time before scraping (ms) */ waitFor?: number; + /** Server-side scrape timeout in milliseconds. */ + timeout?: number; /** Take screenshot */ screenshot?: boolean; /** Take full page screenshot */ @@ -78,6 +81,10 @@ export interface ScrapeOptions { } export interface ScrapeResult { + receipt?: Receipt; + status?: number; + code?: string; + retryAfterSeconds?: number; success: boolean; data?: any; error?: string; diff --git a/src/types/search.ts b/src/types/search.ts index 882873ccbe..d332ecc2e5 100644 --- a/src/types/search.ts +++ b/src/types/search.ts @@ -1,3 +1,4 @@ +import type { Receipt } from '../utils/receipt'; /** * Types for search command */ @@ -122,6 +123,10 @@ export interface SearchResultData { } export interface SearchResult { + receipt?: Receipt; + status?: number; + code?: string; + retryAfterSeconds?: number; success: boolean; data?: SearchResultData; warning?: string; diff --git a/src/utils/client.ts b/src/utils/client.ts index 6519495bfc..fdd24e006a 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -41,8 +41,17 @@ export async function keylessRequest( }); const json: any = await response.json().catch(() => ({})); if (!response.ok) { - throw new Error( - json?.error || `Firecrawl request failed (HTTP ${response.status})` + throw Object.assign( + new Error( + json?.error || `Firecrawl request failed (HTTP ${response.status})` + ), + { + response: { + status: response.status, + data: json, + headers: response.headers, + }, + } ); } return json; diff --git a/src/utils/options.ts b/src/utils/options.ts index a6bc1c933d..3e608f58fb 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -118,6 +118,7 @@ export function parseScrapeOptions(options: any): ScrapeOptions { formats, onlyMainContent: options.onlyMainContent, waitFor: options.waitFor, + timeout: options.timeout, screenshot: options.screenshot, fullPageScreenshot: options.fullPageScreenshot, includeTags: options.includeTags @@ -141,3 +142,13 @@ export function parseScrapeOptions(options: any): ScrapeOptions { redactPII: options.redactPii ?? options.redactPII, }; } + +export function parseScrapeTimeout(value: string): number { + const timeout = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(timeout) || timeout <= 0) { + throw new InvalidArgumentError( + '--timeout must be a positive integer in milliseconds.' + ); + } + return timeout; +} diff --git a/src/utils/output.ts b/src/utils/output.ts index 273e6bc51c..e60b3bc72d 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -180,7 +180,15 @@ export function handleScrapeOutput( if (!result.success) { // Always use stderr for errors to allow piping console.error('Error:', result.error); - process.exit(1); + if (json || pretty || outputPath) { + writeOutput( + JSON.stringify(result, null, pretty ? 2 : undefined), + outputPath, + !!outputPath + ); + } + process.exitCode = 1; + return; } if (!result.data) { @@ -190,6 +198,7 @@ export function handleScrapeOutput( // Determine if we should force JSON output const forceJson = shouldOutputJson(outputPath, json) || + pretty || Array.isArray((result.data as any).tools); // If JSON is forced, always output JSON regardless of format @@ -197,8 +206,18 @@ export function handleScrapeOutput( let jsonContent: string; try { jsonContent = pretty - ? JSON.stringify(result.data, null, 2) - : JSON.stringify(result.data); + ? JSON.stringify( + { + ...result.data, + ...(result.receipt ? { receipt: result.receipt } : {}), + }, + null, + 2 + ) + : JSON.stringify({ + ...result.data, + ...(result.receipt ? { receipt: result.receipt } : {}), + }); } catch (error) { jsonContent = JSON.stringify({ error: 'Failed to serialize response', @@ -246,6 +265,7 @@ export function handleScrapeOutput( outputData = extractMultipleFormats(result.data, formats); } + if (result.receipt) outputData = { ...outputData, receipt: result.receipt }; let jsonContent: string; try { jsonContent = pretty diff --git a/src/utils/receipt.ts b/src/utils/receipt.ts new file mode 100644 index 0000000000..8be60ddf5a --- /dev/null +++ b/src/utils/receipt.ts @@ -0,0 +1,48 @@ +export interface Receipt { + creditsUsed?: number; + requestId?: string; + operationId?: string; + operationType?: 'scrape' | 'search'; +} + +export function receiptFor( + value: any, + operationType: 'scrape' | 'search', + requestId?: string +): Receipt { + const credits = + operationType === 'search' + ? value?.creditsUsed + : (value?.metadata?.creditsUsed ?? + value?.creditsCost ?? + value?.data?.creditsCost); + const operationId = + operationType === 'search' + ? value?.id + : (value?.metadata?.scrapeId ?? value?.scrape_id ?? value?.scrapeId); + return { + ...(typeof credits === 'number' && Number.isFinite(credits) && credits >= 0 + ? { creditsUsed: credits } + : {}), + ...(requestId ? { requestId } : {}), + ...(typeof operationId === 'string' && operationId + ? { operationId, operationType } + : {}), + }; +} + +export function printReceipt(receipt: Receipt, includeRequestId = true): void { + if (includeRequestId && receipt.requestId) + console.error(`Request ID: ${receipt.requestId}`); + if (receipt.operationId) + console.error( + `${receipt.operationType === 'search' ? 'Search' : 'Scrape'} ID: ${receipt.operationId}` + ); + if (receipt.creditsUsed !== undefined) + console.error(`Credits: ${receipt.creditsUsed}`); +} + +export function printRetry(failure: Record): void { + if (typeof failure.retryAfterSeconds === 'number') + console.error(`Retry after: ${failure.retryAfterSeconds}s`); +}