From 85e07362b80857a2a4d5fb488c0f19c4b7623a68 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:54:15 -0400 Subject: [PATCH 1/7] fix(scrape): honor explicit multi-URL JSON output --- README.md | 2 +- skills/firecrawl-scrape/SKILL.md | 4 +- src/__tests__/commands/multi-scrape.test.ts | 143 ++++++++++++++++++++ src/commands/scrape.ts | 45 +++--- src/index.ts | 2 +- 5 files changed, 172 insertions(+), 24 deletions(-) create mode 100644 src/__tests__/commands/multi-scrape.test.ts diff --git a/README.md b/README.md index 2b887933df..a3c2eebf9d 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ When using a custom API URL (anything other than `https://api.firecrawl.dev`), a ### `scrape` - Scrape URLs -Extract content from any webpage. Pass multiple URLs to scrape them concurrently -- each result is saved to `.firecrawl/` automatically. +Extract content from any webpage. Pass multiple URLs to scrape them concurrently. By default, each result is saved to `.firecrawl/`. With `--json` or `-o `, results form one JSON array in input order, written to stdout or the requested file. Each item contains `url`, `success`, and the full `data` (including metadata) or an `error`. `--pretty` indents the array. Any failed URL makes the command exit nonzero, after successful results and errors have been saved. ```bash # Basic usage (outputs markdown) diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 16cfa8a03a..51d9d3e216 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -23,7 +23,7 @@ firecrawl scrape "" --only-main-content -o .firecrawl/page.md # Wait for JS to render, then scrape firecrawl scrape "" --wait-for 3000 -o .firecrawl/page.md -# Multiple URLs (markdown only; each saved to .firecrawl/; -o is ignored) +# Multiple URLs (each saved to .firecrawl/ by default) firecrawl scrape https://example.com https://example.com/blog https://example.com/docs # Get markdown and links together @@ -51,7 +51,7 @@ The cap applies to each PDF, not the whole command or total credits. Extra forma - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). - **Scrape handles static pages and JS-rendered SPAs.** Escalate to `interact` when the page needs interaction (clicks, form fills, pagination) or scrape misses content. -- Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. This mode saves markdown only and ignores `-o`; other requested formats are dropped. If markdown wasn't requested, the whole JSON response is written into the `.md` file. +- Multiple URLs are scraped concurrently. Use `--json` for an ordered JSON array on stdout or `-o results.json` to save it. Each item contains `url`, `success`, and full `data` with metadata or an `error`; any failed URL makes the command exit nonzero. Without either flag, each result is saved under `.firecrawl/` as markdown when available, otherwise JSON in a `.md` file. Check `firecrawl --status` for your concurrency limit. - Single format outputs raw content. Multiple formats (e.g., `--format markdown,links`) output JSON. - Always quote URLs — shell interprets `?` and `&` as special characters. - Naming convention: `.firecrawl/{site}-{path}.md` diff --git a/src/__tests__/commands/multi-scrape.test.ts b/src/__tests__/commands/multi-scrape.test.ts new file mode 100644 index 0000000000..138ef5844a --- /dev/null +++ b/src/__tests__/commands/multi-scrape.test.ts @@ -0,0 +1,143 @@ +import * as fs from 'fs'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { handleMultiScrapeCommand } from '../../commands/scrape'; +import { getClient } from '../../utils/client'; +import { clearInteractSession } from '../../utils/interact-session'; + +vi.mock('../../utils/client', () => ({ + getClient: vi.fn(), + isKeylessMode: () => false, + keylessRequest: vi.fn(), +})); +vi.mock('../../utils/interact-session', () => ({ + saveInteractSession: vi.fn(), + clearInteractSession: vi.fn(), +})); +vi.mock('fs', async () => ({ + ...(await vi.importActual('fs')), + existsSync: vi.fn(() => false), + mkdirSync: vi.fn(), + writeFileSync: vi.fn(), +})); + +const urls = ['https://example.com/a', 'https://example.com/b']; +const documents = urls.map((url, index) => ({ + markdown: `Page ${index}`, + html: `

Page ${index}

`, + metadata: { sourceURL: url, creditsUsed: index + 1, scrapeId: `id-${index}` }, +})); + +describe('multi-URL scrape output', () => { + const scrape = vi.fn(); + let originalExitCode: typeof process.exitCode; + + beforeEach(() => { + originalExitCode = process.exitCode; + process.exitCode = undefined; + vi.clearAllMocks(); + scrape.mockReset(); + vi.mocked(getClient).mockReturnValue({ scrape } as any); + vi.spyOn(process.stdout, 'write').mockReturnValue(true); + vi.spyOn(process.stderr, 'write').mockReturnValue(true); + }); + + afterEach(() => { + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + }); + + it('writes one ordered collection to the requested path despite out-of-order completion', async () => { + let resolveFirst!: (value: unknown) => void; + scrape.mockImplementation(async (url) => { + if (url === urls[0]) + return new Promise((resolve) => { + resolveFirst = resolve; + }); + setImmediate(() => resolveFirst(documents[0])); + return documents[1]; + }); + await handleMultiScrapeCommand(urls, { + url: urls[0], + output: 'out/results.txt', + pretty: true, + }); + expect(fs.writeFileSync).toHaveBeenCalledTimes(1); + const [path, content] = vi.mocked(fs.writeFileSync).mock.calls[0]; + expect(path).toBe('out/results.txt'); + expect(content).toBe( + JSON.stringify( + documents.map((data, index) => ({ + url: urls[index], + success: true, + data, + })), + null, + 2 + ) + ); + expect(fs.mkdirSync).not.toHaveBeenCalledWith( + '.firecrawl', + expect.anything() + ); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(clearInteractSession).toHaveBeenCalledOnce(); + expect(process.exitCode).toBeUndefined(); + }); + + it('prints valid JSON with full metadata on stdout without per-URL files', async () => { + scrape + .mockResolvedValueOnce(documents[0]) + .mockResolvedValueOnce(documents[1]); + await handleMultiScrapeCommand(urls, { url: urls[0], json: true }); + expect(process.stdout.write).toHaveBeenCalledTimes(1); + const output = JSON.parse( + String(vi.mocked(process.stdout.write).mock.calls[0][0]) + ); + expect(output.map((item: any) => item.data)).toEqual(documents); + expect(fs.writeFileSync).not.toHaveBeenCalled(); + expect(fs.mkdirSync).not.toHaveBeenCalled(); + }); + + it.each([false, true])( + 'preserves errors and successful results with failure exit status (all failed: %s)', + async (allFailed) => { + if (allFailed) scrape.mockRejectedValueOnce(new Error('First failed')); + else scrape.mockResolvedValueOnce(documents[0]); + scrape.mockRejectedValueOnce(new Error('Second failed')); + await handleMultiScrapeCommand(urls, { + url: urls[0], + json: true, + output: 'results.json', + }); + const output = JSON.parse( + String(vi.mocked(fs.writeFileSync).mock.calls[0][1]) + ); + expect(output).toEqual([ + allFailed + ? { url: urls[0], success: false, error: 'First failed' } + : { url: urls[0], success: true, data: documents[0] }, + { url: urls[1], success: false, error: 'Second failed' }, + ]); + expect(process.exitCode).toBe(1); + expect(process.stdout.write).not.toHaveBeenCalled(); + } + ); + + it('keeps default per-file behavior and reports partial failure', async () => { + scrape + .mockResolvedValueOnce(documents[0]) + .mockRejectedValueOnce(new Error('Second failed')); + await handleMultiScrapeCommand(urls, { url: urls[0] }); + expect(fs.mkdirSync).toHaveBeenCalledWith('.firecrawl', { + recursive: true, + }); + expect(fs.writeFileSync).toHaveBeenCalledWith( + '.firecrawl/example.com-a.md', + 'Page 0', + 'utf-8' + ); + expect(fs.writeFileSync).toHaveBeenCalledTimes(1); + expect(process.stdout.write).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + }); +}); diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 3dc5d1780b..65220ecc41 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -254,8 +254,7 @@ function urlToFilename(url: string): string { } /** - * Handle scrape for multiple URLs. - * Each result is saved as a separate file in .firecrawl/ + * Explicit output produces an ordered JSON collection; otherwise save per URL. */ export async function handleMultiScrapeCommand( urls: string[], @@ -263,9 +262,9 @@ export async function handleMultiScrapeCommand( ): Promise { const fs = await import('fs'); const path = await import('path'); - + const structuredOutput = !!options.output || !!options.json; const dir = '.firecrawl'; - if (!fs.existsSync(dir)) { + if (!structuredOutput && !fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } @@ -276,9 +275,7 @@ export async function handleMultiScrapeCommand( process.stderr.write(`Scraping ${total} URLs...\n`); const promises = urls.map(async (url) => { - const scrapeOptions: ScrapeOptions = { ...options, url }; - const result = await executeScrape(scrapeOptions); - + const result = await executeScrape({ ...options, url }); const currentCount = ++completedCount; if (!result.success) { @@ -286,33 +283,41 @@ export async function handleMultiScrapeCommand( process.stderr.write( `[${currentCount}/${total}] Error: ${url} - ${result.error}\n` ); - return; + } else if (structuredOutput) { + process.stderr.write(`[${currentCount}/${total}] Scraped: ${url}\n`); + } else { + const filename = urlToFilename(url); + const filepath = path.join(dir, filename); + const content = result.data?.markdown || JSON.stringify(result.data); + fs.writeFileSync(filepath, content, 'utf-8'); + process.stderr.write(`[${currentCount}/${total}] Saved: ${filepath}\n`); } - const filename = urlToFilename(url); - const filepath = path.join(dir, filename); - const content = result.data?.markdown || JSON.stringify(result.data); - fs.writeFileSync(filepath, content, 'utf-8'); - - process.stderr.write(`[${currentCount}/${total}] Saved: ${filepath}\n`); + // Avoid retaining every document in memory for the default per-file mode. + return structuredOutput ? { url, ...result } : undefined; }); - await Promise.all(promises); - + const results = await Promise.all(promises); clearInteractSession(); + + if (structuredOutput) { + writeOutput( + JSON.stringify(results, null, options.pretty ? 2 : undefined), + options.output, + !!options.output + ); + } + process.stderr.write( `\nCompleted: ${completedCount - errorCount}/${total} succeeded` ); if (errorCount > 0) { process.stderr.write(`, ${errorCount} failed`); + process.exitCode = 1; } process.stderr.write( '\nTip: Use --scrape-id with interact to target a specific scrape.\n' ); - - if (errorCount === total) { - process.exit(1); - } } /** diff --git a/src/index.ts b/src/index.ts index ecb1ba8244..c79047f9d3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -356,7 +356,7 @@ program function createScrapeCommand(): Command { const scrapeCmd = new Command('scrape') .description( - 'Scrape one or more URLs. Multiple URLs are scraped concurrently and saved to .firecrawl/' + 'Scrape one or more URLs. Multiple URLs save to .firecrawl/ by default; --json or -o produces an ordered JSON array.' ) .argument('[urls...]', 'URL(s) to scrape') .option( From 62724312b0d0c195a9b8d015e9524ab6b840bca0 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:55:33 -0400 Subject: [PATCH 2/7] fix(search): preserve empty outputs and expose receipts --- src/__tests__/commands/search.test.ts | 97 +++++++++++++++++++++++++++ src/commands/search.ts | 16 +++-- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/src/__tests__/commands/search.test.ts b/src/__tests__/commands/search.test.ts index aeded653fc..c7d70a3b9b 100644 --- a/src/__tests__/commands/search.test.ts +++ b/src/__tests__/commands/search.test.ts @@ -652,6 +652,103 @@ describe('executeSearch', () => { }); }); + describe('output receipts', () => { + let stderr: ReturnType; + + beforeEach(() => { + stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + stderr.mockRestore(); + }); + + it.each([{ json: true }, { pretty: true }])( + 'preserves empty results and receipts in JSON output with %o', + async (flags) => { + mockHttpPost.mockResolvedValue( + mockSearchResponse( + { web: [] }, + { + id: 'search-empty', + creditsUsed: 2, + warning: 'Partial upstream response', + } + ) + ); + + await handleSearchCommand({ + query: 'empty', + output: 'results.json', + ...flags, + }); + + expect(writeOutput).toHaveBeenCalledWith( + expect.any(String), + 'results.json', + true + ); + const body = JSON.parse(vi.mocked(writeOutput).mock.calls[0][0]); + expect(body).toEqual({ + success: true, + data: { web: [] }, + id: 'search-empty', + creditsUsed: 2, + warning: 'Partial upstream response', + }); + expect(stderr).toHaveBeenCalledWith('Search ID: search-empty'); + expect(stderr).toHaveBeenCalledWith('Credits: 2'); + } + ); + + it('honors readable output files for empty results', async () => { + mockHttpPost.mockResolvedValue(mockSearchResponse({})); + await handleSearchCommand({ query: 'empty', output: 'results.txt' }); + expect(writeOutput).toHaveBeenCalledWith( + 'No results found.', + 'results.txt', + true + ); + expect(stderr).not.toHaveBeenCalled(); + }); + + it('prints zero-credit receipts without relabeling search IDs as retry keys', async () => { + mockHttpPost.mockResolvedValue( + mockSearchResponse( + { tools: [] }, + { + id: 'search-free', + creditsUsed: 0, + } + ) + ); + await handleSearchCommand({ query: 'tools', json: true }); + expect(stderr).toHaveBeenCalledWith('Credits: 0'); + const body = JSON.parse(vi.mocked(writeOutput).mock.calls[0][0]); + expect(body.id).toBe('search-free'); + expect(body).not.toHaveProperty('requestId'); + }); + + it('prints receipts for nonempty readable results without mixing them into content', async () => { + mockHttpPost.mockResolvedValue( + mockSearchResponse( + { web: [{ url: 'https://example.com', title: 'Example' }] }, + { + id: 'search-readable', + creditsUsed: 2, + } + ) + ); + await handleSearchCommand({ query: 'example' }); + expect(stderr).toHaveBeenCalledWith('Search ID: search-readable'); + expect(stderr).toHaveBeenCalledWith('Credits: 2'); + const content = vi.mocked(writeOutput).mock.calls[0][0]; + expect(content).toContain('Example'); + expect(content).not.toContain('Search ID:'); + expect(content).not.toContain('Credits:'); + }); + }); + describe('Time-based search parameters', () => { it('should support qdr:h for past hour', async () => { mockHttpPost.mockResolvedValue(mockSearchResponse({ web: [] })); diff --git a/src/commands/search.ts b/src/commands/search.ts index 2bac898562..a3ea5075c3 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -328,6 +328,13 @@ export async function handleSearchCommand( return; } + if (result.id) { + console.error(`Search ID: ${result.id}`); + } + if (typeof result.creditsUsed === 'number') { + console.error(`Credits: ${result.creditsUsed}`); + } + // Check if there are any results const hasResults = (result.data.tools && result.data.tools.length > 0) || @@ -336,11 +343,6 @@ export async function handleSearchCommand( (result.data.news && result.data.news.length > 0) || (result.data.developer && result.data.developer.length > 0); - if (!hasResults && !(result.data.tools && (options.json || options.pretty))) { - console.log('No results found.'); - return; - } - let outputContent: string; // Use JSON format if --json or --pretty flag is set @@ -366,7 +368,9 @@ export async function handleSearchCommand( : JSON.stringify(jsonOutput); } else { // Default to human-readable format - outputContent = formatSearchReadable(result.data, options); + outputContent = hasResults + ? formatSearchReadable(result.data, options) + : 'No results found.'; } writeOutput(outputContent, options.output, !!options.output); From ddc25ec146c64234accb4cb8a8c3f78165587df4 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:56:26 -0400 Subject: [PATCH 3/7] docs: explain search receipts and empty output --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a3c2eebf9d..982a445eaa 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,8 @@ firecrawl search "API documentation" --scrape --scrape-formats markdown,links firecrawl search "AI data tools" ``` +Search prints the returned Search ID and credit usage to stderr when available, keeping stdout suitable for piping. `--json` and `--pretty` preserve the response metadata even when no results match; `-o` also saves empty results. + #### Search Options | Option | Description | From 1a3620ae44642541ed2181a2bacc169555d89840 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:34:15 -0400 Subject: [PATCH 4/7] fix(cli): unify execution receipts and preserve failure diagnostics --- README.md | 65 +++++--- skills/firecrawl-scrape/SKILL.md | 11 ++ skills/firecrawl-search/SKILL.md | 14 +- src/__tests__/alexandria-beta.test.ts | 152 +++++++++++++++++- src/__tests__/commands/multi-scrape.test.ts | 20 ++- src/__tests__/commands/scrape.test.ts | 4 +- src/__tests__/commands/search.test.ts | 13 +- src/__tests__/utils/output.test.ts | 4 +- src/__tests__/utils/receipt.test.ts | 162 ++++++++++++++++++++ src/commands/alexandria.ts | 83 +++++++--- src/commands/scrape.ts | 38 ++++- src/commands/search.ts | 36 +++-- src/index.ts | 11 +- src/types/scrape.ts | 7 + src/types/search.ts | 5 + src/utils/client.ts | 13 +- src/utils/options.ts | 11 ++ src/utils/output.ts | 26 +++- src/utils/receipt.ts | 48 ++++++ 19 files changed, 641 insertions(+), 82 deletions(-) create mode 100644 src/__tests__/utils/receipt.test.ts create mode 100644 src/utils/receipt.ts diff --git a/README.md b/README.md index ee36d6c7ff..e0e04f648a 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ When using a custom API URL (anything other than `https://api.firecrawl.dev`), a ### `scrape` - Scrape URLs -Extract content from any webpage. Pass multiple URLs to scrape them concurrently. By default, each result is saved to `.firecrawl/`. With `--json` or `-o `, results form one JSON array in input order, written to stdout or the requested file. Each item contains `url`, `success`, and the full `data` (including metadata) or an `error`. `--pretty` indents the array. Any failed URL makes the command exit nonzero, after successful results and errors have been saved. +Extract content from any webpage. A single URL writes to stdout unless `-o` is supplied. Pass multiple URLs to scrape them concurrently; without output flags, each result is saved to `.firecrawl/`. With `--json` or `-o `, results form one JSON array in input order, written to stdout or the requested file. Each item contains `url`, `success`, and the full `data` (including metadata) or an `error`. `--pretty` indents the array. Any failed URL makes the command exit nonzero, after successful results and errors have been saved. ```bash # Basic usage (outputs markdown) @@ -234,29 +234,30 @@ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.f #### Scrape Options -| Option | Description | -| -------------------------- | ------------------------------------------------------- | -| `-f, --format ` | Output format(s), comma-separated | -| `-H, --html` | Shortcut for `--format html` | -| `-S, --summary` | Shortcut for `--format summary` | -| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | -| `--wait-for ` | Wait time before scraping (for JS-rendered content) | -| `--screenshot` | Take a screenshot | -| `--full-page-screenshot` | Take a full page screenshot | -| `--include-tags ` | Only include specific HTML tags | -| `--exclude-tags ` | Exclude specific HTML tags | -| `--max-age ` | Maximum age of cached content in milliseconds | -| `--lockdown` | Enable lockdown mode for the scrape | -| `--redact-pii` | Redact personally identifiable information from output | -| `--schema ` | JSON schema for structured extraction | -| `--schema-file ` | Path to JSON schema file for structured extraction | -| `--actions ` | JSON actions array to run during scrape | -| `--actions-file ` | Path to JSON actions file | -| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | -| `-o, --output ` | Save output to file | -| `--json` | Output as JSON format | -| `--pretty` | Pretty print JSON output | -| `--timing` | Show request timing info | +| Option | Description | +| -------------------------- | ------------------------------------------------------------ | +| `-f, --format ` | Output format(s), comma-separated | +| `-H, --html` | Shortcut for `--format html` | +| `-S, --summary` | Shortcut for `--format summary` | +| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | +| `--wait-for ` | Wait time before scraping (for JS-rendered content) | +| `--timeout ` | Request timeout in milliseconds | +| `--screenshot` | Take a screenshot | +| `--full-page-screenshot` | Take a full page screenshot | +| `--include-tags ` | Only include specific HTML tags | +| `--exclude-tags ` | Exclude specific HTML tags | +| `--max-age ` | Maximum cached-content age; use `0` to request fresh content | +| `--lockdown` | Enable lockdown mode for the scrape | +| `--redact-pii` | Redact personally identifiable information from output | +| `--schema ` | JSON schema for structured extraction | +| `--schema-file ` | Path to JSON schema file for structured extraction | +| `--actions ` | JSON actions array to run during scrape | +| `--actions-file ` | Path to JSON actions file | +| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | +| `--pretty` | Pretty print JSON output | +| `--timing` | Show request timing info | #### Available Formats @@ -916,6 +917,22 @@ firecrawl https://example.com -o output.md firecrawl https://example.com --format links --pretty ``` +### Receipts, failures, and retries + +Search, URL scrape, and Alexandria JSON output add a root `receipt` while preserving existing response fields; multi-URL JSON has a `receipt` on each result item. Raw text output stays unchanged. Available receipt fields are: + +- `creditsUsed`: actual credits reported by the response, including zero; absent means unknown, not free. A tool's catalog price is not a charge receipt. +- `requestId`: client idempotency ID for Alexandria calls. +- `operationId` and `operationType` (`search` or `scrape`): the returned server operation ID and its kind, for tracing the operation. + +These identifiers serve different purposes. `--request-id` controls Alexandria retry identity; it is not supported for ordinary URL scrape. Diagnostics print available IDs, credits, and retry timing to stderr. Keep stderr separate from JSON stdout; `2>&1` combines them and is not parseable JSON. + +With `--json` or `-o`, failed search/scrape calls write structured error output before exiting nonzero. A single scrape failure writes JSON even when the requested filename ends in `.md`. Multi-URL explicit output preserves successful results and failures in input order. Inspect the exit code and error fields before treating a saved file as usable content. Missing receipt fields do not establish whether a timed-out operation was billed. + +On a rate limit, wait at least the returned retry delay when available; otherwise use bounded exponential backoff. API keys on the same team share limits, which vary by plan and endpoint. For an unresolved Alexandria request, retain its request ID to recover the same operation. A completed failure can be replayed under the same ID; a deliberately new attempt needs a new ID and may incur a new charge. Do not automatically rotate IDs to bypass a failure. + +For URL scrape, `--timeout ` sets the server-side scrape timeout (the SDK allows transport overhead) and `--max-age 0` requests fresh content. Freshness does not guarantee the source returns a successful page, and a timeout is not proof that upstream work stopped. Inspect returned status/error metadata as well as content. + ### Format Behavior - **Single format**: Outputs raw content (markdown text, HTML, etc.) diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 51d9d3e216..7b4a43520d 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -47,6 +47,17 @@ firecrawl scrape "https://example.com/report.pdf" --max-pages 5 --json -o .firec The cap applies to each PDF, not the whole command or total credits. Extra formats and options can add charges. The CLI does not quote page counts or costs before execution. Use JSON output to inspect the returned `metadata.numPages` (parsed), `metadata.totalPages` (document total), and `metadata.creditsUsed` when present; a smaller parsed count means the result is partial. +## Receipts and recovery + +Use `--json` to preserve metadata and the additive `receipt` (at the root for a single scrape, on each result item for multiple URLs). `receipt.creditsUsed` is actual returned usage, including zero; missing means unknown. Existing `metadata.creditsUsed` remains available when returned. `receipt.operationId` identifies the server scrape; Alexandria `receipt.requestId` is a separate client idempotency ID. Available IDs, credit usage, and retry timing print to stderr. Keep stderr separate from JSON stdout. + +Failures with `--json` or `-o` write structured errors before exiting nonzero, even if the filename ends in `.md`. Inspect the exit code and saved error/status fields before using the content. A successful transport response can still contain a refused or unsuccessful page; do not treat it as task completion or infer a refund. + +- Use `--timeout ` to set the server-side scrape timeout; the SDK allows transport overhead. A timeout does not prove the operation stopped or cost zero credits. +- Use `--max-age 0` when fresh URL content is required. This does not guarantee the source page succeeds. +- On rate limits, honor the returned retry delay when available, otherwise use bounded exponential backoff. Limits are shared across a team's keys and depend on plan and endpoint. +- For Alexandria, keep the same `--request-id` while an operation is unresolved. A completed failure can replay under the same ID; starting a new attempt requires a new ID and may charge again. Never rotate IDs automatically. Ordinary URL scrape does not support `--request-id`. + ## Tips - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 7831635fac..cf3598817d 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -28,7 +28,7 @@ Run `firecrawl search --help` for the full option list. `--categories developer` weighs the developer index beside ordinary web results in this same call (no passage control, no index filters). `--categories research` is a website filter, not the paper index. Dedicated skills: [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) and [firecrawl-research-index](../firecrawl-research-index/SKILL.md). -**Done when:** results are saved under `.firecrawl/`, verified non-empty, processed for the request, and one feedback event is sent within the time window (unless opted out). +**Done when:** the response is saved under `.firecrawl/`, results or an empty result set have been inspected and handled for the request, and eligible feedback is sent within the time window (unless opted out). ## Alexandria in normal search @@ -40,6 +40,14 @@ Use `find-tools` only for an explicitly requested tool set or a missing contract If no returned tool covers the country/market/segment or required inputs, continue with ordinary web results. Do not exhaust the catalogue or pay for adjacent tools just to probe coverage. `--sources web` explicitly opts out of Alexandria; `--sources web --domain-tools` retains domain matches only. +## Receipts and failures + +JSON output preserves the response and additive `receipt`, including empty results. Read `receipt.creditsUsed` for actual reported usage (zero is valid; missing means unknown) and `receipt.operationId` with `operationType: "search"` for the server search ID. Existing `id` and `creditsUsed` fields remain available. Search IDs identify results; they are not Alexandria client idempotency IDs. + +Available IDs, credits, and retry timing print to stderr. Keep stderr separate from JSON stdout. Failed calls with `--json` or `-o` write structured errors before exiting nonzero; check the exit code and `success` before using the file. Empty successful searches still write the requested output and retain their metadata. + +On rate limits, wait at least the returned retry delay when available; otherwise use bounded exponential backoff. API keys on one team share limits, which vary by plan and endpoint. Do not invent a universal requests-per-minute quota or infer zero billing from a missing receipt. + ## Tips - **`--highlights` on by default:** results are query-relevant excerpts, not full-page snippets. Use `--no-highlights` for the original snippets. @@ -66,13 +74,13 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Idempotent:** re-submitting for the same search id returns success but no extra refund. - **`--silent &`** is the right pattern — exit code 0 even on failure, so a rejected/expired call never crashes your pipeline. -Verify the search returned results before reading its `id`. Zero-result searches write no output file, so the file may be missing — or left over from an earlier search. The guard below skips feedback when the file is missing or has zero results; call `search-feedback` only inside it: +Verify the search succeeded before reading its `id`. Empty successful searches preserve their JSON output; failed searches contain error output. The guard below only sends feedback for a successful response with an ID and nonempty results; call `search-feedback` only inside it: ```bash # Send once per search. Rate honestly and replace the placeholder with the # rating that matches what actually happened. The two fields shown # satisfy the substantive-content rule for every rating. -if SEARCH_ID=$(jq -er 'select(any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then +if SEARCH_ID=$(jq -er 'select(.success == true and any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then firecrawl search-feedback "$SEARCH_ID" \ --rating "" \ --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ diff --git a/src/__tests__/alexandria-beta.test.ts b/src/__tests__/alexandria-beta.test.ts index 28a009c7a2..ba5e498577 100644 --- a/src/__tests__/alexandria-beta.test.ts +++ b/src/__tests__/alexandria-beta.test.ts @@ -400,7 +400,14 @@ it('preserves mixed search results, tools and billing metadata', async () => { }; const result = await cli(['search', 'pizza hut', '--json']); expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toEqual(response); + expect(JSON.parse(result.stdout)).toEqual({ + ...response, + receipt: { + creditsUsed: 2, + operationId: 'search-1', + operationType: 'search', + }, + }); expect(requests[0]).toMatchObject({ url: '/v2/search', headers: { authorization: 'Bearer fc-test' }, @@ -431,6 +438,12 @@ it('sends provider calls to Scrape with a stable retry ID and preserves the rece expect(JSON.parse(result.stdout)).toEqual({ ...response, requestId: 'retry-1', + receipt: { + creditsUsed: 1, + requestId: 'retry-1', + operationId: 'scrape-1', + operationType: 'scrape', + }, }); } expect(requests).toHaveLength(2); @@ -535,7 +548,7 @@ it('keeps URL scrape tool contracts in the output', async () => { }; const result = await cli(['scrape', 'https://example.com', '--domain-tools']); expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toEqual(response.data); + expect(JSON.parse(result.stdout)).toEqual({ ...response.data, receipt: {} }); expect(requests[0]).toMatchObject({ url: '/v2/scrape', body: { url: 'https://example.com', domainTools: true }, @@ -898,3 +911,138 @@ it('fails clearly on an unknown thread', async () => { expect(malformed.code).toBe(1); expect(requests).toHaveLength(1); }); + +it('writes a structured plain-scrape refusal to the requested file before exiting', async () => { + status = 403; + response = { + success: false, + error: 'Access denied', + code: 'ACCESS_DENIED', + requestId: 'server-error-1', + }; + const output = join(home, 'refusal.md'); + const result = await cli([ + 'scrape', + 'https://example.com', + '--json', + '-o', + output, + ]); + expect(result.code).toBe(1); + expect(result.stdout).toBe(''); + expect(JSON.parse(readFileSync(output, 'utf8'))).toMatchObject({ + success: false, + error: 'Access denied', + code: 'ACCESS_DENIED', + status: 403, + requestId: 'server-error-1', + }); + expect(requests).toHaveLength(1); +}); + +it('preserves structured retry guidance in failed search JSON and stderr', async () => { + status = 429; + response = { + success: false, + error: 'Rate limited', + code: 'RATE_LIMITED', + retry_after_seconds: 2, + }; + const result = await cli(['search', 'fixture', '--sources', 'web', '--json']); + expect(result.code).toBe(1); + expect(JSON.parse(result.stdout)).toMatchObject({ + success: false, + status: 429, + code: 'RATE_LIMITED', + retryAfterSeconds: 2, + }); + expect(result.stderr).toContain('Retry after: 2s'); +}); + +it('keeps raw scrape stdout pipeable and reports returned cache and charges', async () => { + response = { + success: true, + data: { + markdown: 'fixture', + metadata: { + scrapeId: 'plain-1', + creditsUsed: 0, + cacheState: 'hit', + cachedAt: '2026-09-17T00:00:00Z', + }, + }, + }; + const result = await cli([ + 'scrape', + 'https://example.com', + '--timeout', + '2500', + ]); + expect(result.code).toBe(0); + expect(result.stdout).toBe('fixture\n'); + expect(result.stderr).toContain('Scrape ID: plain-1'); + expect(result.stderr).toContain('Credits: 0'); + expect(result.stderr).toContain('Cache: hit'); + expect(result.stderr).toContain('Cached at: 2026-09-17T00:00:00Z'); + expect(requests[0].body.timeout).toBe(2500); + expect(requests[0].body).not.toHaveProperty('autoResume'); +}); + +it('rejects malformed scrape timeouts before transport', async () => { + for (const timeout of ['0', '-1', '1.5', '10seconds']) { + const result = await cli([ + 'scrape', + 'https://example.com', + '--timeout', + timeout, + ]); + expect(result.code).toBe(1); + expect(result.stderr).toContain('positive integer in milliseconds'); + } + expect(requests).toHaveLength(0); +}); + +it('preserves server retry metadata through the scrape SDK error path', async () => { + status = 429; + response = { + success: false, + error: 'Rate limited', + code: 'RATE_LIMITED', + retry_after_seconds: 3, + }; + const result = await cli(['scrape', 'https://example.com', '--json']); + expect(result.code).toBe(1); + expect(JSON.parse(result.stdout)).toMatchObject({ + success: false, + status: 429, + code: 'RATE_LIMITED', + retryAfterSeconds: 3, + }); + expect(result.stderr).toContain('Retry after: 3s'); +}); + +it('retains query answers and receipts when JSON is explicitly requested', async () => { + response = { + success: true, + data: { + answer: 'fixture answer', + metadata: { scrapeId: 'answer-1', creditsUsed: 2 }, + }, + }; + const result = await cli([ + 'scrape', + 'https://example.com', + '--query', + 'fixture question', + '--json', + ]); + expect(result.code).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + answer: 'fixture answer', + receipt: { + operationId: 'answer-1', + operationType: 'scrape', + creditsUsed: 2, + }, + }); +}); diff --git a/src/__tests__/commands/multi-scrape.test.ts b/src/__tests__/commands/multi-scrape.test.ts index 138ef5844a..5389dd9b44 100644 --- a/src/__tests__/commands/multi-scrape.test.ts +++ b/src/__tests__/commands/multi-scrape.test.ts @@ -70,6 +70,11 @@ describe('multi-URL scrape output', () => { url: urls[index], success: true, data, + receipt: { + creditsUsed: index + 1, + operationId: `id-${index}`, + operationType: 'scrape', + }, })), null, 2 @@ -114,9 +119,18 @@ describe('multi-URL scrape output', () => { ); expect(output).toEqual([ allFailed - ? { url: urls[0], success: false, error: 'First failed' } - : { url: urls[0], success: true, data: documents[0] }, - { url: urls[1], success: false, error: 'Second failed' }, + ? { url: urls[0], success: false, error: 'First failed', receipt: {} } + : { + url: urls[0], + success: true, + data: documents[0], + receipt: { + operationId: 'id-0', + operationType: 'scrape', + creditsUsed: 1, + }, + }, + { url: urls[1], success: false, error: 'Second failed', receipt: {} }, ]); expect(process.exitCode).toBe(1); expect(process.stdout.write).not.toHaveBeenCalled(); diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index b9c9606882..49571d85ac 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -432,6 +432,7 @@ describe('executeScrape', () => { expect(result).toEqual({ success: true, data: mockResponse, + receipt: {}, }); }); @@ -466,6 +467,7 @@ describe('executeScrape', () => { expect(result).toEqual({ success: false, error: errorMessage, + receipt: {}, }); }); @@ -477,7 +479,7 @@ describe('executeScrape', () => { }); expect(result.success).toBe(false); - expect(result.error).toBe('Unknown error occurred'); + expect(result.error).toBe('Request failed'); }); }); diff --git a/src/__tests__/commands/search.test.ts b/src/__tests__/commands/search.test.ts index c7d70a3b9b..f044649914 100644 --- a/src/__tests__/commands/search.test.ts +++ b/src/__tests__/commands/search.test.ts @@ -637,6 +637,7 @@ describe('executeSearch', () => { expect(result).toEqual({ success: false, error: errorMessage, + receipt: {}, }); }); @@ -648,7 +649,7 @@ describe('executeSearch', () => { }); expect(result.success).toBe(false); - expect(result.error).toBe('Unknown error occurred'); + expect(result.error).toBe('Request failed'); }); }); @@ -670,6 +671,11 @@ describe('executeSearch', () => { mockSearchResponse( { web: [] }, { + receipt: { + operationId: 'search-empty', + operationType: 'search', + creditsUsed: 2, + }, id: 'search-empty', creditsUsed: 2, warning: 'Partial upstream response', @@ -692,6 +698,11 @@ describe('executeSearch', () => { expect(body).toEqual({ success: true, data: { web: [] }, + receipt: { + operationId: 'search-empty', + operationType: 'search', + creditsUsed: 2, + }, id: 'search-empty', creditsUsed: 2, warning: 'Partial upstream response', diff --git a/src/__tests__/utils/output.test.ts b/src/__tests__/utils/output.test.ts index 8667768ece..795b9af732 100644 --- a/src/__tests__/utils/output.test.ts +++ b/src/__tests__/utils/output.test.ts @@ -107,7 +107,9 @@ describe('Output Utilities', () => { handleScrapeOutput({ success: false, error: 'API Error' }, ['markdown']); expect(consoleErrorSpy).toHaveBeenCalledWith('Error:', 'API Error'); - expect(processExitSpy).toHaveBeenCalledWith(1); + expect(processExitSpy).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + process.exitCode = undefined; }); it('should output raw markdown for single markdown format', () => { diff --git a/src/__tests__/utils/receipt.test.ts b/src/__tests__/utils/receipt.test.ts new file mode 100644 index 0000000000..c434f753f1 --- /dev/null +++ b/src/__tests__/utils/receipt.test.ts @@ -0,0 +1,162 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { apiFailure } from '../../commands/alexandria'; +import { printReceipt, printRetry, receiptFor } from '../../utils/receipt'; + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe('execution receipts', () => { + it('reports actual charges, including free executions, on each response surface', () => { + expect(receiptFor({ metadata: { creditsUsed: 0 } }, 'scrape')).toEqual({ + creditsUsed: 0, + }); + expect(receiptFor({ creditsCost: 5 }, 'scrape')).toEqual({ + creditsUsed: 5, + }); + expect(receiptFor({ data: { creditsCost: 2.5 } }, 'scrape')).toEqual({ + creditsUsed: 2.5, + }); + expect(receiptFor({ creditsUsed: 0 }, 'search')).toEqual({ + creditsUsed: 0, + }); + }); + + it.each([undefined, null, -1, NaN, Infinity, '5'])( + 'does not present an unknown or invalid charge (%s) as zero', + (creditsUsed) => { + expect( + receiptFor({ metadata: { creditsUsed } }, 'scrape') + ).not.toHaveProperty('creditsUsed'); + expect(receiptFor({ creditsUsed }, 'search')).not.toHaveProperty( + 'creditsUsed' + ); + } + ); + + it('does not turn a quote or a catalogue price into an execution charge', () => { + const quote = { price: 5, estimatedCredits: 5, quote: { creditsCost: 5 } }; + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + printReceipt(receiptFor(quote, 'scrape')); + expect(stderr).not.toHaveBeenCalled(); + }); + + it('keeps the retry request identity distinct from the server operation identity', () => { + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + const stdout = vi + .spyOn(process.stdout, 'write') + .mockImplementation(() => true); + const receipt = receiptFor( + { id: 'search-server-id', creditsUsed: 0 }, + 'search', + 'retry-client-id' + ); + expect(receipt).toMatchObject({ + requestId: 'retry-client-id', + operationId: 'search-server-id', + }); + printReceipt(receipt); + expect(stderr.mock.calls.map(([line]) => line)).toEqual([ + 'Request ID: retry-client-id', + 'Search ID: search-server-id', + 'Credits: 0', + ]); + expect(stdout).not.toHaveBeenCalled(); + }); + + it('can print a server receipt without repeating an already printed request ID', () => { + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + printReceipt( + receiptFor( + { metadata: { scrapeId: 'server-id', creditsUsed: 5 } }, + 'scrape', + 'client-id' + ), + false + ); + expect(stderr.mock.calls.map(([line]) => line)).toEqual([ + 'Scrape ID: server-id', + 'Credits: 5', + ]); + }); +}); + +describe('failure receipts and retry guidance', () => { + it('retains actionable failure fields without serializing transport credentials or unknown fields', () => { + const error = Object.assign(new Error('transport message'), { + config: { headers: { Authorization: 'Bearer secret-key' } }, + response: { + status: 429, + headers: { 'retry-after': '2', 'set-cookie': 'secret-cookie' }, + data: { + error: 'Rate limited', + code: 'rate_limited', + requestId: 'client-id', + scrapeId: 'server-id', + apiKey: 'secret-key', + debug: { headers: { Authorization: 'secret-key' } }, + }, + }, + }); + expect(apiFailure(error)).toEqual({ + success: false, + error: 'Rate limited', + code: 'rate_limited', + requestId: 'client-id', + scrapeId: 'server-id', + status: 429, + retryAfterSeconds: 2, + }); + expect(JSON.stringify(apiFailure(error))).not.toContain('secret'); + }); + + it('rounds a fractional structured retry delay up rather than retrying early', () => { + expect( + apiFailure({ + details: { error: 'Busy', retry_after_seconds: 1.1 }, + status: 429, + }) + ).toMatchObject({ retryAfterSeconds: 2 }); + }); + + it('understands an HTTP-date Retry-After from standard Headers', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-09-17T12:00:00.500Z')); + const failure = apiFailure({ + response: { + status: 503, + data: { error: 'Unavailable' }, + headers: new Headers({ + 'Retry-After': 'Thu, 17 Sep 2026 12:00:03 GMT', + }), + }, + }); + expect(failure.retryAfterSeconds).toBe(3); + }); + + it.each([undefined, 'not-a-delay', '-5', 'Infinity'])( + 'does not invent retry timing from unknown headers (%s)', + (retry) => { + const failure = apiFailure({ + response: { + status: 502, + data: { error: 'Failed' }, + headers: { 'retry-after': retry }, + }, + }); + expect(failure).not.toHaveProperty('retryAfterSeconds'); + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + printRetry(failure); + expect(stderr).not.toHaveBeenCalled(); + } + ); + + it('prints a supplied zero-second retry delay rather than suppressing it', () => { + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + printRetry( + apiFailure({ details: { error: 'Retry now', retryAfterSeconds: 0 } }) + ); + expect(stderr).toHaveBeenCalledWith('Retry after: 0s'); + }); +}); diff --git a/src/commands/alexandria.ts b/src/commands/alexandria.ts index 4766dbd108..77f57caa96 100644 --- a/src/commands/alexandria.ts +++ b/src/commands/alexandria.ts @@ -1,6 +1,7 @@ +import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; import { randomUUID } from 'node:crypto'; import { Command, Option } from 'commander'; -import { SdkError, type AlexandriaCall } from 'firecrawl'; +import { type AlexandriaCall } from 'firecrawl'; import { getClient } from '../utils/client'; import { getApiKey } from '../utils/config'; import { writeOutput } from '../utils/output'; @@ -51,19 +52,41 @@ export function buildCalls(addresses: string[], values: string[] = []): Call[] { }); } -export function apiFailure(error: unknown): Record { +export function apiFailure(error: unknown): Record { + const candidate = error as any; const body = - (error as any)?.response?.data ?? - (error instanceof SdkError - ? { - error: error.message, - code: error.code, - chargeId: error.chargeId, - requiresAction: - (error.details as any)?.requiresAction ?? error.requiresAction, - } - : undefined); - return { + candidate?.response?.data ?? candidate?.details ?? candidate ?? {}; + const headers = candidate?.response?.headers; + const retryHeader = + typeof headers?.get === 'function' + ? headers.get('retry-after') + : headers?.['retry-after']; + const rawRetry = + body?.retry_after_seconds ?? + body?.retryAfterSeconds ?? + body?.details?.retryAfterSeconds ?? + candidate?.retryAfterSeconds ?? + retryHeader; + let retryAfterSeconds: number | undefined; + if ( + typeof rawRetry === 'number' || + (typeof rawRetry === 'string' && /^\d+(\.\d+)?$/.test(rawRetry)) + ) { + const seconds = Number(rawRetry); + if (Number.isFinite(seconds) && seconds >= 0) + retryAfterSeconds = Math.ceil(seconds); + } else if ( + typeof retryHeader === 'string' && + /^[A-Za-z]{3}, /.test(retryHeader) + ) { + const timestamp = Date.parse(retryHeader); + if (Number.isFinite(timestamp)) + retryAfterSeconds = Math.max( + 0, + Math.ceil((timestamp - Date.now()) / 1000) + ); + } + const result: Record = { success: false, error: typeof body?.error === 'string' @@ -71,10 +94,26 @@ export function apiFailure(error: unknown): Record { : error instanceof Error ? error.message : 'Request failed', - ...(typeof body?.code === 'string' && { code: body.code }), - ...(typeof body?.chargeId === 'string' && { chargeId: body.chargeId }), - ...(body?.requiresAction && { requiresAction: body.requiresAction }), }; + for (const key of [ + 'code', + 'chargeId', + 'requestId', + 'scrapeId', + 'scrape_id', + 'id', + ]) { + const value = body?.[key] ?? candidate?.[key]; + if (typeof value === 'string') result[key] = value; + } + const status = + candidate?.response?.status ?? candidate?.status ?? body?.status; + if (typeof status === 'number') result.status = status; + const action = body?.requiresAction ?? candidate?.requiresAction; + if (action) result.requiresAction = action; + if (retryAfterSeconds !== undefined) + result.retryAfterSeconds = retryAfterSeconds; + return result; } export async function requestAlexandria( @@ -107,7 +146,10 @@ export async function requestAlexandria( } catch (error) { envelope = apiFailure(error); } - return { ...envelope, requestId }; + const receipt = receiptFor(envelope, 'scrape', requestId); + printReceipt(receipt, false); + printRetry(envelope); + return { ...envelope, requestId, receipt }; } export async function handleAlexandria( @@ -162,7 +204,10 @@ export function createFindToolsCommand(): Command { '--request ', 'A complete next request returned by Find Tools' ) - .option('--request-id ', 'Reuse for an identical retry') + .option( + '--request-id ', + 'Reuse to recover the same execution; completed results replay' + ) .option('-k, --api-key ', 'Firecrawl API key') .option('--api-url ', 'Firecrawl API URL') .option('-o, --output ', 'Output file') @@ -208,7 +253,7 @@ export function addAlexandriaScrapeOptions(command: Command): void { .addOption( new Option( '--request-id ', - 'Reuse the same ID only for an identical tool retry' + 'Recover the same execution; completed results replay. Never replace an uncertain ID automatically.' ) ) .addOption( diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 65220ecc41..ed91b1c0a0 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -1,3 +1,5 @@ +import { apiFailure } from './alexandria'; +import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; /** * Scrape command implementation */ @@ -115,6 +117,13 @@ export async function executeScrape( scrapeParams.maxAge = options.maxAge; } + if (options.timeout !== undefined) { + if (!Number.isSafeInteger(options.timeout) || options.timeout <= 0) + throw new Error('--timeout must be a positive integer in milliseconds.'); + scrapeParams.timeout = options.timeout; + scrapeParams.autoResume = false; + } + if (options.maxPages !== undefined) { scrapeParams.parsers = [{ type: 'pdf', maxPages: options.maxPages }]; } @@ -155,9 +164,10 @@ export async function executeScrape( if (isKeylessMode(options.apiKey, options.apiUrl)) { // Keyless free tier: header-less request. The API identifies the CLI via // the `integration: 'cli'` field already in scrapeParams. + const { autoResume: _autoResume, ...wireParams } = scrapeParams; const json = await keylessRequest('/v2/scrape', { url: options.url, - ...scrapeParams, + ...wireParams, }); result = json?.data ?? json; } else { @@ -170,9 +180,14 @@ export async function executeScrape( const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime); + const receipt = receiptFor(result, 'scrape'); + printReceipt(receipt); + if (typeof result?.metadata?.cacheState === 'string') + console.error(`Cache: ${result.metadata.cacheState}`); + if (typeof result?.metadata?.cachedAt === 'string') + console.error(`Cached at: ${result.metadata.cachedAt}`); const scrapeId = result?.metadata?.scrapeId; if (scrapeId) { - process.stderr.write(`Scrape ID: ${scrapeId}\n`); try { saveInteractSession({ scrapeId, @@ -190,15 +205,17 @@ export async function executeScrape( return { success: true, data: result, + receipt, }; } catch (error) { const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime, error); - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown error occurred', - }; + const failure = apiFailure(error); + const receipt = receiptFor(failure, 'scrape'); + printReceipt(receipt); + printRetry(failure); + return { ...failure, success: false, receipt }; } } @@ -211,7 +228,14 @@ export async function handleScrapeCommand( const result = await executeScrape(options); // Query mode: output answer directly - if (options.query && result.success && result.data?.answer) { + if ( + options.query && + !options.json && + !options.pretty && + !options.output?.endsWith('.json') && + result.success && + result.data?.answer + ) { writeOutput(result.data.answer, options.output, !!options.output); return; } diff --git a/src/commands/search.ts b/src/commands/search.ts index a3ea5075c3..dd2fd1b15e 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,3 +1,4 @@ +import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; /** * Search command implementation */ @@ -141,16 +142,14 @@ export async function executeSearch( warning: envelope.warning, id: envelope.id, creditsUsed: envelope.creditsUsed, + receipt: receiptFor(envelope, 'search'), }; } catch (error) { + const failure = apiFailure(error); return { + ...failure, success: false, - error: - options.domainTools || options.sources?.includes('alexandria') - ? JSON.stringify(apiFailure(error)) - : error instanceof Error - ? error.message - : 'Unknown error occurred', + receipt: receiptFor(failure, 'search'), }; } } @@ -319,22 +318,24 @@ export async function handleSearchCommand( ): Promise { const result = await executeSearch(options); + printReceipt(result.receipt ?? {}); + printRetry(result as unknown as Record); if (!result.success) { console.error('Error:', result.error); - process.exit(1); + if (options.json || options.pretty || options.output) + writeOutput( + JSON.stringify(result, null, options.pretty ? 2 : undefined), + options.output, + !!options.output + ); + process.exitCode = 1; + return; } if (!result.data) { return; } - if (result.id) { - console.error(`Search ID: ${result.id}`); - } - if (typeof result.creditsUsed === 'number') { - console.error(`Credits: ${result.creditsUsed}`); - } - // Check if there are any results const hasResults = (result.data.tools && result.data.tools.length > 0) || @@ -347,10 +348,15 @@ export async function handleSearchCommand( // Use JSON format if --json or --pretty flag is set // --pretty implies JSON output - if (options.json || options.pretty) { + if ( + options.json || + options.pretty || + options.output?.toLowerCase().endsWith('.json') + ) { const jsonOutput: Record = { success: true, data: result.data, + receipt: result.receipt, }; if (result.warning) { diff --git a/src/index.ts b/src/index.ts index c79047f9d3..0e61810bb7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,7 +71,11 @@ import { handleEnvPullCommand } from './commands/env'; import { handleStatusCommand } from './commands/status'; import { handleDoctorCommand } from './commands/doctor'; import { isUrl, normalizeUrl } from './utils/url'; -import { parseMaxPages, parseScrapeOptions } from './utils/options'; +import { + parseMaxPages, + parseScrapeOptions, + parseScrapeTimeout, +} from './utils/options'; import { isJobId } from './utils/job'; import { ensureAuthenticated, printBanner } from './utils/auth'; import { maybeShowUpdateNotice } from './utils/update-notice'; @@ -373,6 +377,11 @@ function createScrapeCommand(): Command { 'Maximum PDF pages to parse (1-10000). PDFs cost 1 credit per parsed page; extra options may cost more.', parseMaxPages ) + .option( + '--timeout ', + 'Server-side scrape timeout in milliseconds', + parseScrapeTimeout + ) .option('--only-main-content', 'Include only main content', false) .option( '--wait-for ', diff --git a/src/types/scrape.ts b/src/types/scrape.ts index 8f5969163d..591eaf25aa 100644 --- a/src/types/scrape.ts +++ b/src/types/scrape.ts @@ -1,3 +1,4 @@ +import type { Receipt } from '../utils/receipt'; /** * Types and interfaces for the scrape command */ @@ -32,6 +33,8 @@ export interface ScrapeOptions { onlyMainContent?: boolean; /** Wait time before scraping (ms) */ waitFor?: number; + /** Server-side scrape timeout in milliseconds. */ + timeout?: number; /** Take screenshot */ screenshot?: boolean; /** Take full page screenshot */ @@ -78,6 +81,10 @@ export interface ScrapeOptions { } export interface ScrapeResult { + receipt?: Receipt; + status?: number; + code?: string; + retryAfterSeconds?: number; success: boolean; data?: any; error?: string; diff --git a/src/types/search.ts b/src/types/search.ts index 882873ccbe..d332ecc2e5 100644 --- a/src/types/search.ts +++ b/src/types/search.ts @@ -1,3 +1,4 @@ +import type { Receipt } from '../utils/receipt'; /** * Types for search command */ @@ -122,6 +123,10 @@ export interface SearchResultData { } export interface SearchResult { + receipt?: Receipt; + status?: number; + code?: string; + retryAfterSeconds?: number; success: boolean; data?: SearchResultData; warning?: string; diff --git a/src/utils/client.ts b/src/utils/client.ts index 6519495bfc..fdd24e006a 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -41,8 +41,17 @@ export async function keylessRequest( }); const json: any = await response.json().catch(() => ({})); if (!response.ok) { - throw new Error( - json?.error || `Firecrawl request failed (HTTP ${response.status})` + throw Object.assign( + new Error( + json?.error || `Firecrawl request failed (HTTP ${response.status})` + ), + { + response: { + status: response.status, + data: json, + headers: response.headers, + }, + } ); } return json; diff --git a/src/utils/options.ts b/src/utils/options.ts index a6bc1c933d..3e608f58fb 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -118,6 +118,7 @@ export function parseScrapeOptions(options: any): ScrapeOptions { formats, onlyMainContent: options.onlyMainContent, waitFor: options.waitFor, + timeout: options.timeout, screenshot: options.screenshot, fullPageScreenshot: options.fullPageScreenshot, includeTags: options.includeTags @@ -141,3 +142,13 @@ export function parseScrapeOptions(options: any): ScrapeOptions { redactPII: options.redactPii ?? options.redactPII, }; } + +export function parseScrapeTimeout(value: string): number { + const timeout = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(timeout) || timeout <= 0) { + throw new InvalidArgumentError( + '--timeout must be a positive integer in milliseconds.' + ); + } + return timeout; +} diff --git a/src/utils/output.ts b/src/utils/output.ts index 273e6bc51c..e60b3bc72d 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -180,7 +180,15 @@ export function handleScrapeOutput( if (!result.success) { // Always use stderr for errors to allow piping console.error('Error:', result.error); - process.exit(1); + if (json || pretty || outputPath) { + writeOutput( + JSON.stringify(result, null, pretty ? 2 : undefined), + outputPath, + !!outputPath + ); + } + process.exitCode = 1; + return; } if (!result.data) { @@ -190,6 +198,7 @@ export function handleScrapeOutput( // Determine if we should force JSON output const forceJson = shouldOutputJson(outputPath, json) || + pretty || Array.isArray((result.data as any).tools); // If JSON is forced, always output JSON regardless of format @@ -197,8 +206,18 @@ export function handleScrapeOutput( let jsonContent: string; try { jsonContent = pretty - ? JSON.stringify(result.data, null, 2) - : JSON.stringify(result.data); + ? JSON.stringify( + { + ...result.data, + ...(result.receipt ? { receipt: result.receipt } : {}), + }, + null, + 2 + ) + : JSON.stringify({ + ...result.data, + ...(result.receipt ? { receipt: result.receipt } : {}), + }); } catch (error) { jsonContent = JSON.stringify({ error: 'Failed to serialize response', @@ -246,6 +265,7 @@ export function handleScrapeOutput( outputData = extractMultipleFormats(result.data, formats); } + if (result.receipt) outputData = { ...outputData, receipt: result.receipt }; let jsonContent: string; try { jsonContent = pretty diff --git a/src/utils/receipt.ts b/src/utils/receipt.ts new file mode 100644 index 0000000000..8be60ddf5a --- /dev/null +++ b/src/utils/receipt.ts @@ -0,0 +1,48 @@ +export interface Receipt { + creditsUsed?: number; + requestId?: string; + operationId?: string; + operationType?: 'scrape' | 'search'; +} + +export function receiptFor( + value: any, + operationType: 'scrape' | 'search', + requestId?: string +): Receipt { + const credits = + operationType === 'search' + ? value?.creditsUsed + : (value?.metadata?.creditsUsed ?? + value?.creditsCost ?? + value?.data?.creditsCost); + const operationId = + operationType === 'search' + ? value?.id + : (value?.metadata?.scrapeId ?? value?.scrape_id ?? value?.scrapeId); + return { + ...(typeof credits === 'number' && Number.isFinite(credits) && credits >= 0 + ? { creditsUsed: credits } + : {}), + ...(requestId ? { requestId } : {}), + ...(typeof operationId === 'string' && operationId + ? { operationId, operationType } + : {}), + }; +} + +export function printReceipt(receipt: Receipt, includeRequestId = true): void { + if (includeRequestId && receipt.requestId) + console.error(`Request ID: ${receipt.requestId}`); + if (receipt.operationId) + console.error( + `${receipt.operationType === 'search' ? 'Search' : 'Scrape'} ID: ${receipt.operationId}` + ); + if (receipt.creditsUsed !== undefined) + console.error(`Credits: ${receipt.creditsUsed}`); +} + +export function printRetry(failure: Record): void { + if (typeof failure.retryAfterSeconds === 'number') + console.error(`Retry after: ${failure.retryAfterSeconds}s`); +} From 08e04f2f44fedee13ac0d5937ceeeea7654540d9 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:36:31 -0400 Subject: [PATCH 5/7] fix(cli): expose search pretty output and clarify failed saves --- README.md | 3 ++- skills/firecrawl-scrape/SKILL.md | 4 ++-- src/__tests__/alexandria-beta.test.ts | 28 +++++++++++++++++++++++++++ src/index.ts | 6 +----- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e0e04f648a..770ee6dcb5 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ When using a custom API URL (anything other than `https://api.firecrawl.dev`), a ### `scrape` - Scrape URLs -Extract content from any webpage. A single URL writes to stdout unless `-o` is supplied. Pass multiple URLs to scrape them concurrently; without output flags, each result is saved to `.firecrawl/`. With `--json` or `-o `, results form one JSON array in input order, written to stdout or the requested file. Each item contains `url`, `success`, and the full `data` (including metadata) or an `error`. `--pretty` indents the array. Any failed URL makes the command exit nonzero, after successful results and errors have been saved. +Extract content from any webpage. A single URL writes to stdout unless `-o` is supplied. Pass multiple URLs to scrape them concurrently; without output flags, successful results are saved to `.firecrawl/`, while failed URLs are reported on stderr without per-URL files. With `--json` or `-o `, results form one JSON array in input order, written to stdout or the requested file. Each item contains `url`, `success`, and the full `data` (including metadata) or an `error`. `--pretty` indents the array. Any failed URL makes the command exit nonzero, after successful results and errors have been saved. ```bash # Basic usage (outputs markdown) @@ -367,6 +367,7 @@ Search prints the returned Search ID and credit usage to stderr when available, | `--only-main-content` | Include only main content when scraping (default: true) | | `-o, --output ` | Save to file | | `--json` | Output as compact JSON | +| `--pretty` | Output as pretty-printed JSON | #### Examples diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 7b4a43520d..3f92247e36 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -23,7 +23,7 @@ firecrawl scrape "" --only-main-content -o .firecrawl/page.md # Wait for JS to render, then scrape firecrawl scrape "" --wait-for 3000 -o .firecrawl/page.md -# Multiple URLs (each saved to .firecrawl/ by default) +# Multiple URLs (successful results saved to .firecrawl/; failures reported) firecrawl scrape https://example.com https://example.com/blog https://example.com/docs # Get markdown and links together @@ -62,7 +62,7 @@ Failures with `--json` or `-o` write structured errors before exiting nonzero, e - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). - **Scrape handles static pages and JS-rendered SPAs.** Escalate to `interact` when the page needs interaction (clicks, form fills, pagination) or scrape misses content. -- Multiple URLs are scraped concurrently. Use `--json` for an ordered JSON array on stdout or `-o results.json` to save it. Each item contains `url`, `success`, and full `data` with metadata or an `error`; any failed URL makes the command exit nonzero. Without either flag, each result is saved under `.firecrawl/` as markdown when available, otherwise JSON in a `.md` file. Check `firecrawl --status` for your concurrency limit. +- Multiple URLs are scraped concurrently. Use `--json` for an ordered JSON array on stdout or `-o results.json` to save it. Each item contains `url`, `success`, and full `data` with metadata or an `error`; any failed URL makes the command exit nonzero. Without either flag, successful results are saved under `.firecrawl/` as markdown when available, otherwise JSON in a `.md` file. Failed URLs are reported on stderr without creating per-URL files. Check `firecrawl --status` for your concurrency limit. - Single format outputs raw content. Multiple formats (e.g., `--format markdown,links`) output JSON. - Always quote URLs — shell interprets `?` and `&` as special characters. - Naming convention: `.firecrawl/{site}-{path}.md` diff --git a/src/__tests__/alexandria-beta.test.ts b/src/__tests__/alexandria-beta.test.ts index ba5e498577..54d8d990c2 100644 --- a/src/__tests__/alexandria-beta.test.ts +++ b/src/__tests__/alexandria-beta.test.ts @@ -1046,3 +1046,31 @@ it('retains query answers and receipts when JSON is explicitly requested', async }, }); }); + +it('accepts search --pretty and preserves empty results and receipts', async () => { + response = { + success: true, + id: 'pretty-search', + creditsUsed: 0, + data: { web: [] }, + }; + const result = await cli([ + 'search', + 'fixture', + '--sources', + 'web', + '--pretty', + ]); + expect(result.code).toBe(0); + expect(result.stdout).toContain('\n "success": true'); + expect(JSON.parse(result.stdout)).toMatchObject({ + ...response, + receipt: { + creditsUsed: 0, + operationId: 'pretty-search', + operationType: 'search', + }, + }); + expect(result.stderr).toContain('Search ID: pretty-search'); + expect(requests).toHaveLength(1); +}); diff --git a/src/index.ts b/src/index.ts index 0e61810bb7..a93d7cdd17 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1009,12 +1009,8 @@ function createSearchCommand(): Command { ) .option('--api-url ', 'API URL (overrides global --api-url)') .option('-o, --output ', 'Output file path (default: stdout)') - // .option( - // '-p, --pretty', - // 'Output as pretty JSON (default: human-readable)', - // false - // ) .option('--json', 'Output as compact JSON', false) + .option('--pretty', 'Output as pretty-printed JSON', false) .action(async (query, options) => { // Parse sources let sources: SearchSource[] = ['web', 'alexandria']; From cba416fe4f337794108380c27096ea2903fe0bcd Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:40:17 -0400 Subject: [PATCH 6/7] refactor: scope PR to Alexandria receipts and retry guidance --- README.md | 74 +++---- skills/firecrawl-scrape/SKILL.md | 15 +- skills/firecrawl-search/SKILL.md | 14 +- src/__tests__/alexandria-beta.test.ts | 174 +--------------- src/__tests__/commands/multi-scrape.test.ts | 157 --------------- src/__tests__/commands/scrape.test.ts | 4 +- src/__tests__/commands/search.test.ts | 110 +---------- src/__tests__/utils/output.test.ts | 4 +- src/__tests__/utils/receipt.test.ts | 208 +++++--------------- src/commands/scrape.ts | 83 +++----- src/commands/search.ts | 38 ++-- src/index.ts | 19 +- src/types/scrape.ts | 7 - src/types/search.ts | 5 - src/utils/client.ts | 13 +- src/utils/options.ts | 11 -- src/utils/output.ts | 26 +-- 17 files changed, 146 insertions(+), 816 deletions(-) delete mode 100644 src/__tests__/commands/multi-scrape.test.ts diff --git a/README.md b/README.md index 770ee6dcb5..3caf3bb47b 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ When using a custom API URL (anything other than `https://api.firecrawl.dev`), a ### `scrape` - Scrape URLs -Extract content from any webpage. A single URL writes to stdout unless `-o` is supplied. Pass multiple URLs to scrape them concurrently; without output flags, successful results are saved to `.firecrawl/`, while failed URLs are reported on stderr without per-URL files. With `--json` or `-o `, results form one JSON array in input order, written to stdout or the requested file. Each item contains `url`, `success`, and the full `data` (including metadata) or an `error`. `--pretty` indents the array. Any failed URL makes the command exit nonzero, after successful results and errors have been saved. +Extract content from any webpage. Pass multiple URLs to scrape them concurrently -- each result is saved to `.firecrawl/` automatically. ```bash # Basic usage (outputs markdown) @@ -234,30 +234,29 @@ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.f #### Scrape Options -| Option | Description | -| -------------------------- | ------------------------------------------------------------ | -| `-f, --format ` | Output format(s), comma-separated | -| `-H, --html` | Shortcut for `--format html` | -| `-S, --summary` | Shortcut for `--format summary` | -| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | -| `--wait-for ` | Wait time before scraping (for JS-rendered content) | -| `--timeout ` | Request timeout in milliseconds | -| `--screenshot` | Take a screenshot | -| `--full-page-screenshot` | Take a full page screenshot | -| `--include-tags ` | Only include specific HTML tags | -| `--exclude-tags ` | Exclude specific HTML tags | -| `--max-age ` | Maximum cached-content age; use `0` to request fresh content | -| `--lockdown` | Enable lockdown mode for the scrape | -| `--redact-pii` | Redact personally identifiable information from output | -| `--schema ` | JSON schema for structured extraction | -| `--schema-file ` | Path to JSON schema file for structured extraction | -| `--actions ` | JSON actions array to run during scrape | -| `--actions-file ` | Path to JSON actions file | -| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | -| `-o, --output ` | Save output to file | -| `--json` | Output as JSON format | -| `--pretty` | Pretty print JSON output | -| `--timing` | Show request timing info | +| Option | Description | +| -------------------------- | ------------------------------------------------------- | +| `-f, --format ` | Output format(s), comma-separated | +| `-H, --html` | Shortcut for `--format html` | +| `-S, --summary` | Shortcut for `--format summary` | +| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | +| `--wait-for ` | Wait time before scraping (for JS-rendered content) | +| `--screenshot` | Take a screenshot | +| `--full-page-screenshot` | Take a full page screenshot | +| `--include-tags ` | Only include specific HTML tags | +| `--exclude-tags ` | Exclude specific HTML tags | +| `--max-age ` | Maximum age of cached content in milliseconds | +| `--lockdown` | Enable lockdown mode for the scrape | +| `--redact-pii` | Redact personally identifiable information from output | +| `--schema ` | JSON schema for structured extraction | +| `--schema-file ` | Path to JSON schema file for structured extraction | +| `--actions ` | JSON actions array to run during scrape | +| `--actions-file ` | Path to JSON actions file | +| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | +| `--pretty` | Pretty print JSON output | +| `--timing` | Show request timing info | #### Available Formats @@ -346,8 +345,6 @@ firecrawl search "API documentation" --scrape --scrape-formats markdown,links firecrawl search "AI data tools" ``` -Search prints the returned Search ID and credit usage to stderr when available, keeping stdout suitable for piping. `--json` and `--pretty` preserve the response metadata even when no results match; `-o` also saves empty results. - #### Search Options | Option | Description | @@ -367,7 +364,6 @@ Search prints the returned Search ID and credit usage to stderr when available, | `--only-main-content` | Include only main content when scraping (default: true) | | `-o, --output ` | Save to file | | `--json` | Output as compact JSON | -| `--pretty` | Output as pretty-printed JSON | #### Examples @@ -918,22 +914,6 @@ firecrawl https://example.com -o output.md firecrawl https://example.com --format links --pretty ``` -### Receipts, failures, and retries - -Search, URL scrape, and Alexandria JSON output add a root `receipt` while preserving existing response fields; multi-URL JSON has a `receipt` on each result item. Raw text output stays unchanged. Available receipt fields are: - -- `creditsUsed`: actual credits reported by the response, including zero; absent means unknown, not free. A tool's catalog price is not a charge receipt. -- `requestId`: client idempotency ID for Alexandria calls. -- `operationId` and `operationType` (`search` or `scrape`): the returned server operation ID and its kind, for tracing the operation. - -These identifiers serve different purposes. `--request-id` controls Alexandria retry identity; it is not supported for ordinary URL scrape. Diagnostics print available IDs, credits, and retry timing to stderr. Keep stderr separate from JSON stdout; `2>&1` combines them and is not parseable JSON. - -With `--json` or `-o`, failed search/scrape calls write structured error output before exiting nonzero. A single scrape failure writes JSON even when the requested filename ends in `.md`. Multi-URL explicit output preserves successful results and failures in input order. Inspect the exit code and error fields before treating a saved file as usable content. Missing receipt fields do not establish whether a timed-out operation was billed. - -On a rate limit, wait at least the returned retry delay when available; otherwise use bounded exponential backoff. API keys on the same team share limits, which vary by plan and endpoint. For an unresolved Alexandria request, retain its request ID to recover the same operation. A completed failure can be replayed under the same ID; a deliberately new attempt needs a new ID and may incur a new charge. Do not automatically rotate IDs to bypass a failure. - -For URL scrape, `--timeout ` sets the server-side scrape timeout (the SDK allows transport overhead) and `--max-age 0` requests fresh content. Freshness does not guarantee the source returns a successful page, and a timeout is not proof that upstream work stopped. Inspect returned status/error metadata as well as content. - ### Format Behavior - **Single format**: Outputs raw content (markdown text, HTML, etc.) @@ -1059,3 +1039,9 @@ https://www.firecrawl.dev/app/settings?tab=data-sources. Never infer consent fro failed lookup or automatically retry an acceptance. The API remains authoritative for organization access and acceptance authority. After confirmed success, rerun the original provider command; its normal credits apply. + +### Alexandria receipts and retries + +Alexandria execution JSON includes an additive `receipt`: `creditsUsed` is actual reported usage (missing means unknown), `requestId` is the client idempotency identity, and `operationId`/`operationType` identify the server scrape. Existing response fields remain available. IDs, reported credits, and available retry delays print to stderr. + +Use the same request ID to recover pending or uncertain execution. Completed results, including failures, replay under the same ID; a deliberate new execution needs a new ID and may charge again. Never automatically rotate an uncertain ID. Structured failures preserve available status, code, action and retry metadata. diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 3f92247e36..16cfa8a03a 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -23,7 +23,7 @@ firecrawl scrape "" --only-main-content -o .firecrawl/page.md # Wait for JS to render, then scrape firecrawl scrape "" --wait-for 3000 -o .firecrawl/page.md -# Multiple URLs (successful results saved to .firecrawl/; failures reported) +# Multiple URLs (markdown only; each saved to .firecrawl/; -o is ignored) firecrawl scrape https://example.com https://example.com/blog https://example.com/docs # Get markdown and links together @@ -47,22 +47,11 @@ firecrawl scrape "https://example.com/report.pdf" --max-pages 5 --json -o .firec The cap applies to each PDF, not the whole command or total credits. Extra formats and options can add charges. The CLI does not quote page counts or costs before execution. Use JSON output to inspect the returned `metadata.numPages` (parsed), `metadata.totalPages` (document total), and `metadata.creditsUsed` when present; a smaller parsed count means the result is partial. -## Receipts and recovery - -Use `--json` to preserve metadata and the additive `receipt` (at the root for a single scrape, on each result item for multiple URLs). `receipt.creditsUsed` is actual returned usage, including zero; missing means unknown. Existing `metadata.creditsUsed` remains available when returned. `receipt.operationId` identifies the server scrape; Alexandria `receipt.requestId` is a separate client idempotency ID. Available IDs, credit usage, and retry timing print to stderr. Keep stderr separate from JSON stdout. - -Failures with `--json` or `-o` write structured errors before exiting nonzero, even if the filename ends in `.md`. Inspect the exit code and saved error/status fields before using the content. A successful transport response can still contain a refused or unsuccessful page; do not treat it as task completion or infer a refund. - -- Use `--timeout ` to set the server-side scrape timeout; the SDK allows transport overhead. A timeout does not prove the operation stopped or cost zero credits. -- Use `--max-age 0` when fresh URL content is required. This does not guarantee the source page succeeds. -- On rate limits, honor the returned retry delay when available, otherwise use bounded exponential backoff. Limits are shared across a team's keys and depend on plan and endpoint. -- For Alexandria, keep the same `--request-id` while an operation is unresolved. A completed failure can replay under the same ID; starting a new attempt requires a new ID and may charge again. Never rotate IDs automatically. Ordinary URL scrape does not support `--request-id`. - ## Tips - **Prefer plain scrape over `--query`.** Scrape to a file, then use `grep`, `head`, or read the markdown directly — you can search and reason over the full content yourself. Use `--query` only when you want a single targeted answer without saving the page (costs 5 extra credits). - **Scrape handles static pages and JS-rendered SPAs.** Escalate to `interact` when the page needs interaction (clicks, form fills, pagination) or scrape misses content. -- Multiple URLs are scraped concurrently. Use `--json` for an ordered JSON array on stdout or `-o results.json` to save it. Each item contains `url`, `success`, and full `data` with metadata or an `error`; any failed URL makes the command exit nonzero. Without either flag, successful results are saved under `.firecrawl/` as markdown when available, otherwise JSON in a `.md` file. Failed URLs are reported on stderr without creating per-URL files. Check `firecrawl --status` for your concurrency limit. +- Multiple URLs are scraped concurrently — check `firecrawl --status` for your concurrency limit. This mode saves markdown only and ignores `-o`; other requested formats are dropped. If markdown wasn't requested, the whole JSON response is written into the `.md` file. - Single format outputs raw content. Multiple formats (e.g., `--format markdown,links`) output JSON. - Always quote URLs — shell interprets `?` and `&` as special characters. - Naming convention: `.firecrawl/{site}-{path}.md` diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index cf3598817d..7831635fac 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -28,7 +28,7 @@ Run `firecrawl search --help` for the full option list. `--categories developer` weighs the developer index beside ordinary web results in this same call (no passage control, no index filters). `--categories research` is a website filter, not the paper index. Dedicated skills: [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) and [firecrawl-research-index](../firecrawl-research-index/SKILL.md). -**Done when:** the response is saved under `.firecrawl/`, results or an empty result set have been inspected and handled for the request, and eligible feedback is sent within the time window (unless opted out). +**Done when:** results are saved under `.firecrawl/`, verified non-empty, processed for the request, and one feedback event is sent within the time window (unless opted out). ## Alexandria in normal search @@ -40,14 +40,6 @@ Use `find-tools` only for an explicitly requested tool set or a missing contract If no returned tool covers the country/market/segment or required inputs, continue with ordinary web results. Do not exhaust the catalogue or pay for adjacent tools just to probe coverage. `--sources web` explicitly opts out of Alexandria; `--sources web --domain-tools` retains domain matches only. -## Receipts and failures - -JSON output preserves the response and additive `receipt`, including empty results. Read `receipt.creditsUsed` for actual reported usage (zero is valid; missing means unknown) and `receipt.operationId` with `operationType: "search"` for the server search ID. Existing `id` and `creditsUsed` fields remain available. Search IDs identify results; they are not Alexandria client idempotency IDs. - -Available IDs, credits, and retry timing print to stderr. Keep stderr separate from JSON stdout. Failed calls with `--json` or `-o` write structured errors before exiting nonzero; check the exit code and `success` before using the file. Empty successful searches still write the requested output and retain their metadata. - -On rate limits, wait at least the returned retry delay when available; otherwise use bounded exponential backoff. API keys on one team share limits, which vary by plan and endpoint. Do not invent a universal requests-per-minute quota or infer zero billing from a missing receipt. - ## Tips - **`--highlights` on by default:** results are query-relevant excerpts, not full-page snippets. Use `--no-highlights` for the original snippets. @@ -74,13 +66,13 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Idempotent:** re-submitting for the same search id returns success but no extra refund. - **`--silent &`** is the right pattern — exit code 0 even on failure, so a rejected/expired call never crashes your pipeline. -Verify the search succeeded before reading its `id`. Empty successful searches preserve their JSON output; failed searches contain error output. The guard below only sends feedback for a successful response with an ID and nonempty results; call `search-feedback` only inside it: +Verify the search returned results before reading its `id`. Zero-result searches write no output file, so the file may be missing — or left over from an earlier search. The guard below skips feedback when the file is missing or has zero results; call `search-feedback` only inside it: ```bash # Send once per search. Rate honestly and replace the placeholder with the # rating that matches what actually happened. The two fields shown # satisfy the substantive-content rule for every rating. -if SEARCH_ID=$(jq -er 'select(.success == true and any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then +if SEARCH_ID=$(jq -er 'select(any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then firecrawl search-feedback "$SEARCH_ID" \ --rating "" \ --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ diff --git a/src/__tests__/alexandria-beta.test.ts b/src/__tests__/alexandria-beta.test.ts index 54d8d990c2..83d7332bbf 100644 --- a/src/__tests__/alexandria-beta.test.ts +++ b/src/__tests__/alexandria-beta.test.ts @@ -400,14 +400,7 @@ it('preserves mixed search results, tools and billing metadata', async () => { }; const result = await cli(['search', 'pizza hut', '--json']); expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toEqual({ - ...response, - receipt: { - creditsUsed: 2, - operationId: 'search-1', - operationType: 'search', - }, - }); + expect(JSON.parse(result.stdout)).toEqual(response); expect(requests[0]).toMatchObject({ url: '/v2/search', headers: { authorization: 'Bearer fc-test' }, @@ -548,7 +541,7 @@ it('keeps URL scrape tool contracts in the output', async () => { }; const result = await cli(['scrape', 'https://example.com', '--domain-tools']); expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toEqual({ ...response.data, receipt: {} }); + expect(JSON.parse(result.stdout)).toEqual(response.data); expect(requests[0]).toMatchObject({ url: '/v2/scrape', body: { url: 'https://example.com', domainTools: true }, @@ -911,166 +904,3 @@ it('fails clearly on an unknown thread', async () => { expect(malformed.code).toBe(1); expect(requests).toHaveLength(1); }); - -it('writes a structured plain-scrape refusal to the requested file before exiting', async () => { - status = 403; - response = { - success: false, - error: 'Access denied', - code: 'ACCESS_DENIED', - requestId: 'server-error-1', - }; - const output = join(home, 'refusal.md'); - const result = await cli([ - 'scrape', - 'https://example.com', - '--json', - '-o', - output, - ]); - expect(result.code).toBe(1); - expect(result.stdout).toBe(''); - expect(JSON.parse(readFileSync(output, 'utf8'))).toMatchObject({ - success: false, - error: 'Access denied', - code: 'ACCESS_DENIED', - status: 403, - requestId: 'server-error-1', - }); - expect(requests).toHaveLength(1); -}); - -it('preserves structured retry guidance in failed search JSON and stderr', async () => { - status = 429; - response = { - success: false, - error: 'Rate limited', - code: 'RATE_LIMITED', - retry_after_seconds: 2, - }; - const result = await cli(['search', 'fixture', '--sources', 'web', '--json']); - expect(result.code).toBe(1); - expect(JSON.parse(result.stdout)).toMatchObject({ - success: false, - status: 429, - code: 'RATE_LIMITED', - retryAfterSeconds: 2, - }); - expect(result.stderr).toContain('Retry after: 2s'); -}); - -it('keeps raw scrape stdout pipeable and reports returned cache and charges', async () => { - response = { - success: true, - data: { - markdown: 'fixture', - metadata: { - scrapeId: 'plain-1', - creditsUsed: 0, - cacheState: 'hit', - cachedAt: '2026-09-17T00:00:00Z', - }, - }, - }; - const result = await cli([ - 'scrape', - 'https://example.com', - '--timeout', - '2500', - ]); - expect(result.code).toBe(0); - expect(result.stdout).toBe('fixture\n'); - expect(result.stderr).toContain('Scrape ID: plain-1'); - expect(result.stderr).toContain('Credits: 0'); - expect(result.stderr).toContain('Cache: hit'); - expect(result.stderr).toContain('Cached at: 2026-09-17T00:00:00Z'); - expect(requests[0].body.timeout).toBe(2500); - expect(requests[0].body).not.toHaveProperty('autoResume'); -}); - -it('rejects malformed scrape timeouts before transport', async () => { - for (const timeout of ['0', '-1', '1.5', '10seconds']) { - const result = await cli([ - 'scrape', - 'https://example.com', - '--timeout', - timeout, - ]); - expect(result.code).toBe(1); - expect(result.stderr).toContain('positive integer in milliseconds'); - } - expect(requests).toHaveLength(0); -}); - -it('preserves server retry metadata through the scrape SDK error path', async () => { - status = 429; - response = { - success: false, - error: 'Rate limited', - code: 'RATE_LIMITED', - retry_after_seconds: 3, - }; - const result = await cli(['scrape', 'https://example.com', '--json']); - expect(result.code).toBe(1); - expect(JSON.parse(result.stdout)).toMatchObject({ - success: false, - status: 429, - code: 'RATE_LIMITED', - retryAfterSeconds: 3, - }); - expect(result.stderr).toContain('Retry after: 3s'); -}); - -it('retains query answers and receipts when JSON is explicitly requested', async () => { - response = { - success: true, - data: { - answer: 'fixture answer', - metadata: { scrapeId: 'answer-1', creditsUsed: 2 }, - }, - }; - const result = await cli([ - 'scrape', - 'https://example.com', - '--query', - 'fixture question', - '--json', - ]); - expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toMatchObject({ - answer: 'fixture answer', - receipt: { - operationId: 'answer-1', - operationType: 'scrape', - creditsUsed: 2, - }, - }); -}); - -it('accepts search --pretty and preserves empty results and receipts', async () => { - response = { - success: true, - id: 'pretty-search', - creditsUsed: 0, - data: { web: [] }, - }; - const result = await cli([ - 'search', - 'fixture', - '--sources', - 'web', - '--pretty', - ]); - expect(result.code).toBe(0); - expect(result.stdout).toContain('\n "success": true'); - expect(JSON.parse(result.stdout)).toMatchObject({ - ...response, - receipt: { - creditsUsed: 0, - operationId: 'pretty-search', - operationType: 'search', - }, - }); - expect(result.stderr).toContain('Search ID: pretty-search'); - expect(requests).toHaveLength(1); -}); diff --git a/src/__tests__/commands/multi-scrape.test.ts b/src/__tests__/commands/multi-scrape.test.ts deleted file mode 100644 index 5389dd9b44..0000000000 --- a/src/__tests__/commands/multi-scrape.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -import * as fs from 'fs'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { handleMultiScrapeCommand } from '../../commands/scrape'; -import { getClient } from '../../utils/client'; -import { clearInteractSession } from '../../utils/interact-session'; - -vi.mock('../../utils/client', () => ({ - getClient: vi.fn(), - isKeylessMode: () => false, - keylessRequest: vi.fn(), -})); -vi.mock('../../utils/interact-session', () => ({ - saveInteractSession: vi.fn(), - clearInteractSession: vi.fn(), -})); -vi.mock('fs', async () => ({ - ...(await vi.importActual('fs')), - existsSync: vi.fn(() => false), - mkdirSync: vi.fn(), - writeFileSync: vi.fn(), -})); - -const urls = ['https://example.com/a', 'https://example.com/b']; -const documents = urls.map((url, index) => ({ - markdown: `Page ${index}`, - html: `

Page ${index}

`, - metadata: { sourceURL: url, creditsUsed: index + 1, scrapeId: `id-${index}` }, -})); - -describe('multi-URL scrape output', () => { - const scrape = vi.fn(); - let originalExitCode: typeof process.exitCode; - - beforeEach(() => { - originalExitCode = process.exitCode; - process.exitCode = undefined; - vi.clearAllMocks(); - scrape.mockReset(); - vi.mocked(getClient).mockReturnValue({ scrape } as any); - vi.spyOn(process.stdout, 'write').mockReturnValue(true); - vi.spyOn(process.stderr, 'write').mockReturnValue(true); - }); - - afterEach(() => { - process.exitCode = originalExitCode; - vi.restoreAllMocks(); - }); - - it('writes one ordered collection to the requested path despite out-of-order completion', async () => { - let resolveFirst!: (value: unknown) => void; - scrape.mockImplementation(async (url) => { - if (url === urls[0]) - return new Promise((resolve) => { - resolveFirst = resolve; - }); - setImmediate(() => resolveFirst(documents[0])); - return documents[1]; - }); - await handleMultiScrapeCommand(urls, { - url: urls[0], - output: 'out/results.txt', - pretty: true, - }); - expect(fs.writeFileSync).toHaveBeenCalledTimes(1); - const [path, content] = vi.mocked(fs.writeFileSync).mock.calls[0]; - expect(path).toBe('out/results.txt'); - expect(content).toBe( - JSON.stringify( - documents.map((data, index) => ({ - url: urls[index], - success: true, - data, - receipt: { - creditsUsed: index + 1, - operationId: `id-${index}`, - operationType: 'scrape', - }, - })), - null, - 2 - ) - ); - expect(fs.mkdirSync).not.toHaveBeenCalledWith( - '.firecrawl', - expect.anything() - ); - expect(process.stdout.write).not.toHaveBeenCalled(); - expect(clearInteractSession).toHaveBeenCalledOnce(); - expect(process.exitCode).toBeUndefined(); - }); - - it('prints valid JSON with full metadata on stdout without per-URL files', async () => { - scrape - .mockResolvedValueOnce(documents[0]) - .mockResolvedValueOnce(documents[1]); - await handleMultiScrapeCommand(urls, { url: urls[0], json: true }); - expect(process.stdout.write).toHaveBeenCalledTimes(1); - const output = JSON.parse( - String(vi.mocked(process.stdout.write).mock.calls[0][0]) - ); - expect(output.map((item: any) => item.data)).toEqual(documents); - expect(fs.writeFileSync).not.toHaveBeenCalled(); - expect(fs.mkdirSync).not.toHaveBeenCalled(); - }); - - it.each([false, true])( - 'preserves errors and successful results with failure exit status (all failed: %s)', - async (allFailed) => { - if (allFailed) scrape.mockRejectedValueOnce(new Error('First failed')); - else scrape.mockResolvedValueOnce(documents[0]); - scrape.mockRejectedValueOnce(new Error('Second failed')); - await handleMultiScrapeCommand(urls, { - url: urls[0], - json: true, - output: 'results.json', - }); - const output = JSON.parse( - String(vi.mocked(fs.writeFileSync).mock.calls[0][1]) - ); - expect(output).toEqual([ - allFailed - ? { url: urls[0], success: false, error: 'First failed', receipt: {} } - : { - url: urls[0], - success: true, - data: documents[0], - receipt: { - operationId: 'id-0', - operationType: 'scrape', - creditsUsed: 1, - }, - }, - { url: urls[1], success: false, error: 'Second failed', receipt: {} }, - ]); - expect(process.exitCode).toBe(1); - expect(process.stdout.write).not.toHaveBeenCalled(); - } - ); - - it('keeps default per-file behavior and reports partial failure', async () => { - scrape - .mockResolvedValueOnce(documents[0]) - .mockRejectedValueOnce(new Error('Second failed')); - await handleMultiScrapeCommand(urls, { url: urls[0] }); - expect(fs.mkdirSync).toHaveBeenCalledWith('.firecrawl', { - recursive: true, - }); - expect(fs.writeFileSync).toHaveBeenCalledWith( - '.firecrawl/example.com-a.md', - 'Page 0', - 'utf-8' - ); - expect(fs.writeFileSync).toHaveBeenCalledTimes(1); - expect(process.stdout.write).not.toHaveBeenCalled(); - expect(process.exitCode).toBe(1); - }); -}); diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index 49571d85ac..b9c9606882 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -432,7 +432,6 @@ describe('executeScrape', () => { expect(result).toEqual({ success: true, data: mockResponse, - receipt: {}, }); }); @@ -467,7 +466,6 @@ describe('executeScrape', () => { expect(result).toEqual({ success: false, error: errorMessage, - receipt: {}, }); }); @@ -479,7 +477,7 @@ describe('executeScrape', () => { }); expect(result.success).toBe(false); - expect(result.error).toBe('Request failed'); + expect(result.error).toBe('Unknown error occurred'); }); }); diff --git a/src/__tests__/commands/search.test.ts b/src/__tests__/commands/search.test.ts index f044649914..aeded653fc 100644 --- a/src/__tests__/commands/search.test.ts +++ b/src/__tests__/commands/search.test.ts @@ -637,7 +637,6 @@ describe('executeSearch', () => { expect(result).toEqual({ success: false, error: errorMessage, - receipt: {}, }); }); @@ -649,114 +648,7 @@ describe('executeSearch', () => { }); expect(result.success).toBe(false); - expect(result.error).toBe('Request failed'); - }); - }); - - describe('output receipts', () => { - let stderr: ReturnType; - - beforeEach(() => { - stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); - }); - - afterEach(() => { - stderr.mockRestore(); - }); - - it.each([{ json: true }, { pretty: true }])( - 'preserves empty results and receipts in JSON output with %o', - async (flags) => { - mockHttpPost.mockResolvedValue( - mockSearchResponse( - { web: [] }, - { - receipt: { - operationId: 'search-empty', - operationType: 'search', - creditsUsed: 2, - }, - id: 'search-empty', - creditsUsed: 2, - warning: 'Partial upstream response', - } - ) - ); - - await handleSearchCommand({ - query: 'empty', - output: 'results.json', - ...flags, - }); - - expect(writeOutput).toHaveBeenCalledWith( - expect.any(String), - 'results.json', - true - ); - const body = JSON.parse(vi.mocked(writeOutput).mock.calls[0][0]); - expect(body).toEqual({ - success: true, - data: { web: [] }, - receipt: { - operationId: 'search-empty', - operationType: 'search', - creditsUsed: 2, - }, - id: 'search-empty', - creditsUsed: 2, - warning: 'Partial upstream response', - }); - expect(stderr).toHaveBeenCalledWith('Search ID: search-empty'); - expect(stderr).toHaveBeenCalledWith('Credits: 2'); - } - ); - - it('honors readable output files for empty results', async () => { - mockHttpPost.mockResolvedValue(mockSearchResponse({})); - await handleSearchCommand({ query: 'empty', output: 'results.txt' }); - expect(writeOutput).toHaveBeenCalledWith( - 'No results found.', - 'results.txt', - true - ); - expect(stderr).not.toHaveBeenCalled(); - }); - - it('prints zero-credit receipts without relabeling search IDs as retry keys', async () => { - mockHttpPost.mockResolvedValue( - mockSearchResponse( - { tools: [] }, - { - id: 'search-free', - creditsUsed: 0, - } - ) - ); - await handleSearchCommand({ query: 'tools', json: true }); - expect(stderr).toHaveBeenCalledWith('Credits: 0'); - const body = JSON.parse(vi.mocked(writeOutput).mock.calls[0][0]); - expect(body.id).toBe('search-free'); - expect(body).not.toHaveProperty('requestId'); - }); - - it('prints receipts for nonempty readable results without mixing them into content', async () => { - mockHttpPost.mockResolvedValue( - mockSearchResponse( - { web: [{ url: 'https://example.com', title: 'Example' }] }, - { - id: 'search-readable', - creditsUsed: 2, - } - ) - ); - await handleSearchCommand({ query: 'example' }); - expect(stderr).toHaveBeenCalledWith('Search ID: search-readable'); - expect(stderr).toHaveBeenCalledWith('Credits: 2'); - const content = vi.mocked(writeOutput).mock.calls[0][0]; - expect(content).toContain('Example'); - expect(content).not.toContain('Search ID:'); - expect(content).not.toContain('Credits:'); + expect(result.error).toBe('Unknown error occurred'); }); }); diff --git a/src/__tests__/utils/output.test.ts b/src/__tests__/utils/output.test.ts index 795b9af732..8667768ece 100644 --- a/src/__tests__/utils/output.test.ts +++ b/src/__tests__/utils/output.test.ts @@ -107,9 +107,7 @@ describe('Output Utilities', () => { handleScrapeOutput({ success: false, error: 'API Error' }, ['markdown']); expect(consoleErrorSpy).toHaveBeenCalledWith('Error:', 'API Error'); - expect(processExitSpy).not.toHaveBeenCalled(); - expect(process.exitCode).toBe(1); - process.exitCode = undefined; + expect(processExitSpy).toHaveBeenCalledWith(1); }); it('should output raw markdown for single markdown format', () => { diff --git a/src/__tests__/utils/receipt.test.ts b/src/__tests__/utils/receipt.test.ts index c434f753f1..9b8510b9f9 100644 --- a/src/__tests__/utils/receipt.test.ts +++ b/src/__tests__/utils/receipt.test.ts @@ -1,162 +1,60 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, expect, it, vi } from 'vitest'; import { apiFailure } from '../../commands/alexandria'; -import { printReceipt, printRetry, receiptFor } from '../../utils/receipt'; - -afterEach(() => { - vi.restoreAllMocks(); - vi.useRealTimers(); +import { receiptFor } from '../../utils/receipt'; + +afterEach(() => vi.useRealTimers()); + +it('keeps actual zero charges and separates client identity from the server operation', () => { + expect( + receiptFor( + { scrape_id: 'server-1', data: { creditsCost: 0 } }, + 'scrape', + 'client-1' + ) + ).toEqual({ + creditsUsed: 0, + requestId: 'client-1', + operationId: 'server-1', + operationType: 'scrape', + }); + expect(receiptFor({}, 'scrape')).toEqual({}); + expect( + receiptFor({ metadata: { creditsUsed: -1 } }, 'scrape') + ).not.toHaveProperty('creditsUsed'); }); -describe('execution receipts', () => { - it('reports actual charges, including free executions, on each response surface', () => { - expect(receiptFor({ metadata: { creditsUsed: 0 } }, 'scrape')).toEqual({ - creditsUsed: 0, - }); - expect(receiptFor({ creditsCost: 5 }, 'scrape')).toEqual({ - creditsUsed: 5, - }); - expect(receiptFor({ data: { creditsCost: 2.5 } }, 'scrape')).toEqual({ - creditsUsed: 2.5, - }); - expect(receiptFor({ creditsUsed: 0 }, 'search')).toEqual({ - creditsUsed: 0, - }); - }); - - it.each([undefined, null, -1, NaN, Infinity, '5'])( - 'does not present an unknown or invalid charge (%s) as zero', - (creditsUsed) => { - expect( - receiptFor({ metadata: { creditsUsed } }, 'scrape') - ).not.toHaveProperty('creditsUsed'); - expect(receiptFor({ creditsUsed }, 'search')).not.toHaveProperty( - 'creditsUsed' - ); - } - ); - - it('does not turn a quote or a catalogue price into an execution charge', () => { - const quote = { price: 5, estimatedCredits: 5, quote: { creditsCost: 5 } }; - const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); - printReceipt(receiptFor(quote, 'scrape')); - expect(stderr).not.toHaveBeenCalled(); - }); - - it('keeps the retry request identity distinct from the server operation identity', () => { - const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); - const stdout = vi - .spyOn(process.stdout, 'write') - .mockImplementation(() => true); - const receipt = receiptFor( - { id: 'search-server-id', creditsUsed: 0 }, - 'search', - 'retry-client-id' - ); - expect(receipt).toMatchObject({ - requestId: 'retry-client-id', - operationId: 'search-server-id', - }); - printReceipt(receipt); - expect(stderr.mock.calls.map(([line]) => line)).toEqual([ - 'Request ID: retry-client-id', - 'Search ID: search-server-id', - 'Credits: 0', - ]); - expect(stdout).not.toHaveBeenCalled(); - }); - - it('can print a server receipt without repeating an already printed request ID', () => { - const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); - printReceipt( - receiptFor( - { metadata: { scrapeId: 'server-id', creditsUsed: 5 } }, - 'scrape', - 'client-id' - ), - false - ); - expect(stderr.mock.calls.map(([line]) => line)).toEqual([ - 'Scrape ID: server-id', - 'Credits: 5', - ]); - }); -}); - -describe('failure receipts and retry guidance', () => { - it('retains actionable failure fields without serializing transport credentials or unknown fields', () => { - const error = Object.assign(new Error('transport message'), { - config: { headers: { Authorization: 'Bearer secret-key' } }, - response: { - status: 429, - headers: { 'retry-after': '2', 'set-cookie': 'secret-cookie' }, - data: { - error: 'Rate limited', - code: 'rate_limited', - requestId: 'client-id', - scrapeId: 'server-id', - apiKey: 'secret-key', - debug: { headers: { Authorization: 'secret-key' } }, - }, - }, - }); - expect(apiFailure(error)).toEqual({ - success: false, - error: 'Rate limited', - code: 'rate_limited', - requestId: 'client-id', - scrapeId: 'server-id', +it('preserves actionable errors without serializing transport credentials', () => { + const failure = apiFailure({ + response: { status: 429, - retryAfterSeconds: 2, - }); - expect(JSON.stringify(apiFailure(error))).not.toContain('secret'); - }); - - it('rounds a fractional structured retry delay up rather than retrying early', () => { - expect( - apiFailure({ - details: { error: 'Busy', retry_after_seconds: 1.1 }, - status: 429, - }) - ).toMatchObject({ retryAfterSeconds: 2 }); - }); - - it('understands an HTTP-date Retry-After from standard Headers', () => { - vi.useFakeTimers(); - vi.setSystemTime(new Date('2026-09-17T12:00:00.500Z')); - const failure = apiFailure({ - response: { - status: 503, - data: { error: 'Unavailable' }, - headers: new Headers({ - 'Retry-After': 'Thu, 17 Sep 2026 12:00:03 GMT', - }), + data: { + error: 'Limited', + code: 'RATE_LIMITED', + requestId: 'request-1', + retry_after_seconds: 1.5, }, - }); - expect(failure.retryAfterSeconds).toBe(3); - }); - - it.each([undefined, 'not-a-delay', '-5', 'Infinity'])( - 'does not invent retry timing from unknown headers (%s)', - (retry) => { - const failure = apiFailure({ - response: { - status: 502, - data: { error: 'Failed' }, - headers: { 'retry-after': retry }, - }, - }); - expect(failure).not.toHaveProperty('retryAfterSeconds'); - const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); - printRetry(failure); - expect(stderr).not.toHaveBeenCalled(); - } - ); + config: { headers: { authorization: 'secret' } }, + }, + }); + expect(failure).toEqual({ + success: false, + error: 'Limited', + code: 'RATE_LIMITED', + requestId: 'request-1', + status: 429, + retryAfterSeconds: 2, + }); + expect(JSON.stringify(failure)).not.toContain('secret'); +}); - it('prints a supplied zero-second retry delay rather than suppressing it', () => { - const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); - printRetry( - apiFailure({ details: { error: 'Retry now', retryAfterSeconds: 0 } }) - ); - expect(stderr).toHaveBeenCalledWith('Retry after: 0s'); - }); +it('handles HTTP-date retry delays without treating invalid numeric delays as dates', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-09-17T00:00:00Z')); + const failure = (retry: string) => + apiFailure({ + response: { status: 429, headers: { 'retry-after': retry } }, + }); + expect(failure('Thu, 17 Sep 2026 00:00:03 GMT').retryAfterSeconds).toBe(3); + for (const value of ['-5', 'nonsense', '']) + expect(failure(value)).not.toHaveProperty('retryAfterSeconds'); }); diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index ed91b1c0a0..3dc5d1780b 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -1,5 +1,3 @@ -import { apiFailure } from './alexandria'; -import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; /** * Scrape command implementation */ @@ -117,13 +115,6 @@ export async function executeScrape( scrapeParams.maxAge = options.maxAge; } - if (options.timeout !== undefined) { - if (!Number.isSafeInteger(options.timeout) || options.timeout <= 0) - throw new Error('--timeout must be a positive integer in milliseconds.'); - scrapeParams.timeout = options.timeout; - scrapeParams.autoResume = false; - } - if (options.maxPages !== undefined) { scrapeParams.parsers = [{ type: 'pdf', maxPages: options.maxPages }]; } @@ -164,10 +155,9 @@ export async function executeScrape( if (isKeylessMode(options.apiKey, options.apiUrl)) { // Keyless free tier: header-less request. The API identifies the CLI via // the `integration: 'cli'` field already in scrapeParams. - const { autoResume: _autoResume, ...wireParams } = scrapeParams; const json = await keylessRequest('/v2/scrape', { url: options.url, - ...wireParams, + ...scrapeParams, }); result = json?.data ?? json; } else { @@ -180,14 +170,9 @@ export async function executeScrape( const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime); - const receipt = receiptFor(result, 'scrape'); - printReceipt(receipt); - if (typeof result?.metadata?.cacheState === 'string') - console.error(`Cache: ${result.metadata.cacheState}`); - if (typeof result?.metadata?.cachedAt === 'string') - console.error(`Cached at: ${result.metadata.cachedAt}`); const scrapeId = result?.metadata?.scrapeId; if (scrapeId) { + process.stderr.write(`Scrape ID: ${scrapeId}\n`); try { saveInteractSession({ scrapeId, @@ -205,17 +190,15 @@ export async function executeScrape( return { success: true, data: result, - receipt, }; } catch (error) { const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime, error); - const failure = apiFailure(error); - const receipt = receiptFor(failure, 'scrape'); - printReceipt(receipt); - printRetry(failure); - return { ...failure, success: false, receipt }; + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error occurred', + }; } } @@ -228,14 +211,7 @@ export async function handleScrapeCommand( const result = await executeScrape(options); // Query mode: output answer directly - if ( - options.query && - !options.json && - !options.pretty && - !options.output?.endsWith('.json') && - result.success && - result.data?.answer - ) { + if (options.query && result.success && result.data?.answer) { writeOutput(result.data.answer, options.output, !!options.output); return; } @@ -278,7 +254,8 @@ function urlToFilename(url: string): string { } /** - * Explicit output produces an ordered JSON collection; otherwise save per URL. + * Handle scrape for multiple URLs. + * Each result is saved as a separate file in .firecrawl/ */ export async function handleMultiScrapeCommand( urls: string[], @@ -286,9 +263,9 @@ export async function handleMultiScrapeCommand( ): Promise { const fs = await import('fs'); const path = await import('path'); - const structuredOutput = !!options.output || !!options.json; + const dir = '.firecrawl'; - if (!structuredOutput && !fs.existsSync(dir)) { + if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } @@ -299,7 +276,9 @@ export async function handleMultiScrapeCommand( process.stderr.write(`Scraping ${total} URLs...\n`); const promises = urls.map(async (url) => { - const result = await executeScrape({ ...options, url }); + const scrapeOptions: ScrapeOptions = { ...options, url }; + const result = await executeScrape(scrapeOptions); + const currentCount = ++completedCount; if (!result.success) { @@ -307,41 +286,33 @@ export async function handleMultiScrapeCommand( process.stderr.write( `[${currentCount}/${total}] Error: ${url} - ${result.error}\n` ); - } else if (structuredOutput) { - process.stderr.write(`[${currentCount}/${total}] Scraped: ${url}\n`); - } else { - const filename = urlToFilename(url); - const filepath = path.join(dir, filename); - const content = result.data?.markdown || JSON.stringify(result.data); - fs.writeFileSync(filepath, content, 'utf-8'); - process.stderr.write(`[${currentCount}/${total}] Saved: ${filepath}\n`); + return; } - // Avoid retaining every document in memory for the default per-file mode. - return structuredOutput ? { url, ...result } : undefined; - }); + const filename = urlToFilename(url); + const filepath = path.join(dir, filename); + const content = result.data?.markdown || JSON.stringify(result.data); + fs.writeFileSync(filepath, content, 'utf-8'); - const results = await Promise.all(promises); - clearInteractSession(); + process.stderr.write(`[${currentCount}/${total}] Saved: ${filepath}\n`); + }); - if (structuredOutput) { - writeOutput( - JSON.stringify(results, null, options.pretty ? 2 : undefined), - options.output, - !!options.output - ); - } + await Promise.all(promises); + clearInteractSession(); process.stderr.write( `\nCompleted: ${completedCount - errorCount}/${total} succeeded` ); if (errorCount > 0) { process.stderr.write(`, ${errorCount} failed`); - process.exitCode = 1; } process.stderr.write( '\nTip: Use --scrape-id with interact to target a specific scrape.\n' ); + + if (errorCount === total) { + process.exit(1); + } } /** diff --git a/src/commands/search.ts b/src/commands/search.ts index dd2fd1b15e..2bac898562 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,4 +1,3 @@ -import { receiptFor, printReceipt, printRetry } from '../utils/receipt'; /** * Search command implementation */ @@ -142,14 +141,16 @@ export async function executeSearch( warning: envelope.warning, id: envelope.id, creditsUsed: envelope.creditsUsed, - receipt: receiptFor(envelope, 'search'), }; } catch (error) { - const failure = apiFailure(error); return { - ...failure, success: false, - receipt: receiptFor(failure, 'search'), + error: + options.domainTools || options.sources?.includes('alexandria') + ? JSON.stringify(apiFailure(error)) + : error instanceof Error + ? error.message + : 'Unknown error occurred', }; } } @@ -318,18 +319,9 @@ export async function handleSearchCommand( ): Promise { const result = await executeSearch(options); - printReceipt(result.receipt ?? {}); - printRetry(result as unknown as Record); if (!result.success) { console.error('Error:', result.error); - if (options.json || options.pretty || options.output) - writeOutput( - JSON.stringify(result, null, options.pretty ? 2 : undefined), - options.output, - !!options.output - ); - process.exitCode = 1; - return; + process.exit(1); } if (!result.data) { @@ -344,19 +336,19 @@ export async function handleSearchCommand( (result.data.news && result.data.news.length > 0) || (result.data.developer && result.data.developer.length > 0); + if (!hasResults && !(result.data.tools && (options.json || options.pretty))) { + console.log('No results found.'); + return; + } + let outputContent: string; // Use JSON format if --json or --pretty flag is set // --pretty implies JSON output - if ( - options.json || - options.pretty || - options.output?.toLowerCase().endsWith('.json') - ) { + if (options.json || options.pretty) { const jsonOutput: Record = { success: true, data: result.data, - receipt: result.receipt, }; if (result.warning) { @@ -374,9 +366,7 @@ export async function handleSearchCommand( : JSON.stringify(jsonOutput); } else { // Default to human-readable format - outputContent = hasResults - ? formatSearchReadable(result.data, options) - : 'No results found.'; + outputContent = formatSearchReadable(result.data, options); } writeOutput(outputContent, options.output, !!options.output); diff --git a/src/index.ts b/src/index.ts index a93d7cdd17..ecb1ba8244 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,11 +71,7 @@ import { handleEnvPullCommand } from './commands/env'; import { handleStatusCommand } from './commands/status'; import { handleDoctorCommand } from './commands/doctor'; import { isUrl, normalizeUrl } from './utils/url'; -import { - parseMaxPages, - parseScrapeOptions, - parseScrapeTimeout, -} from './utils/options'; +import { parseMaxPages, parseScrapeOptions } from './utils/options'; import { isJobId } from './utils/job'; import { ensureAuthenticated, printBanner } from './utils/auth'; import { maybeShowUpdateNotice } from './utils/update-notice'; @@ -360,7 +356,7 @@ program function createScrapeCommand(): Command { const scrapeCmd = new Command('scrape') .description( - 'Scrape one or more URLs. Multiple URLs save to .firecrawl/ by default; --json or -o produces an ordered JSON array.' + 'Scrape one or more URLs. Multiple URLs are scraped concurrently and saved to .firecrawl/' ) .argument('[urls...]', 'URL(s) to scrape') .option( @@ -377,11 +373,6 @@ function createScrapeCommand(): Command { 'Maximum PDF pages to parse (1-10000). PDFs cost 1 credit per parsed page; extra options may cost more.', parseMaxPages ) - .option( - '--timeout ', - 'Server-side scrape timeout in milliseconds', - parseScrapeTimeout - ) .option('--only-main-content', 'Include only main content', false) .option( '--wait-for ', @@ -1009,8 +1000,12 @@ function createSearchCommand(): Command { ) .option('--api-url ', 'API URL (overrides global --api-url)') .option('-o, --output ', 'Output file path (default: stdout)') + // .option( + // '-p, --pretty', + // 'Output as pretty JSON (default: human-readable)', + // false + // ) .option('--json', 'Output as compact JSON', false) - .option('--pretty', 'Output as pretty-printed JSON', false) .action(async (query, options) => { // Parse sources let sources: SearchSource[] = ['web', 'alexandria']; diff --git a/src/types/scrape.ts b/src/types/scrape.ts index 591eaf25aa..8f5969163d 100644 --- a/src/types/scrape.ts +++ b/src/types/scrape.ts @@ -1,4 +1,3 @@ -import type { Receipt } from '../utils/receipt'; /** * Types and interfaces for the scrape command */ @@ -33,8 +32,6 @@ export interface ScrapeOptions { onlyMainContent?: boolean; /** Wait time before scraping (ms) */ waitFor?: number; - /** Server-side scrape timeout in milliseconds. */ - timeout?: number; /** Take screenshot */ screenshot?: boolean; /** Take full page screenshot */ @@ -81,10 +78,6 @@ export interface ScrapeOptions { } export interface ScrapeResult { - receipt?: Receipt; - status?: number; - code?: string; - retryAfterSeconds?: number; success: boolean; data?: any; error?: string; diff --git a/src/types/search.ts b/src/types/search.ts index d332ecc2e5..882873ccbe 100644 --- a/src/types/search.ts +++ b/src/types/search.ts @@ -1,4 +1,3 @@ -import type { Receipt } from '../utils/receipt'; /** * Types for search command */ @@ -123,10 +122,6 @@ export interface SearchResultData { } export interface SearchResult { - receipt?: Receipt; - status?: number; - code?: string; - retryAfterSeconds?: number; success: boolean; data?: SearchResultData; warning?: string; diff --git a/src/utils/client.ts b/src/utils/client.ts index fdd24e006a..6519495bfc 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -41,17 +41,8 @@ export async function keylessRequest( }); const json: any = await response.json().catch(() => ({})); if (!response.ok) { - throw Object.assign( - new Error( - json?.error || `Firecrawl request failed (HTTP ${response.status})` - ), - { - response: { - status: response.status, - data: json, - headers: response.headers, - }, - } + throw new Error( + json?.error || `Firecrawl request failed (HTTP ${response.status})` ); } return json; diff --git a/src/utils/options.ts b/src/utils/options.ts index 3e608f58fb..a6bc1c933d 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -118,7 +118,6 @@ export function parseScrapeOptions(options: any): ScrapeOptions { formats, onlyMainContent: options.onlyMainContent, waitFor: options.waitFor, - timeout: options.timeout, screenshot: options.screenshot, fullPageScreenshot: options.fullPageScreenshot, includeTags: options.includeTags @@ -142,13 +141,3 @@ export function parseScrapeOptions(options: any): ScrapeOptions { redactPII: options.redactPii ?? options.redactPII, }; } - -export function parseScrapeTimeout(value: string): number { - const timeout = Number(value); - if (!/^\d+$/.test(value) || !Number.isSafeInteger(timeout) || timeout <= 0) { - throw new InvalidArgumentError( - '--timeout must be a positive integer in milliseconds.' - ); - } - return timeout; -} diff --git a/src/utils/output.ts b/src/utils/output.ts index e60b3bc72d..273e6bc51c 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -180,15 +180,7 @@ export function handleScrapeOutput( if (!result.success) { // Always use stderr for errors to allow piping console.error('Error:', result.error); - if (json || pretty || outputPath) { - writeOutput( - JSON.stringify(result, null, pretty ? 2 : undefined), - outputPath, - !!outputPath - ); - } - process.exitCode = 1; - return; + process.exit(1); } if (!result.data) { @@ -198,7 +190,6 @@ export function handleScrapeOutput( // Determine if we should force JSON output const forceJson = shouldOutputJson(outputPath, json) || - pretty || Array.isArray((result.data as any).tools); // If JSON is forced, always output JSON regardless of format @@ -206,18 +197,8 @@ export function handleScrapeOutput( let jsonContent: string; try { jsonContent = pretty - ? JSON.stringify( - { - ...result.data, - ...(result.receipt ? { receipt: result.receipt } : {}), - }, - null, - 2 - ) - : JSON.stringify({ - ...result.data, - ...(result.receipt ? { receipt: result.receipt } : {}), - }); + ? JSON.stringify(result.data, null, 2) + : JSON.stringify(result.data); } catch (error) { jsonContent = JSON.stringify({ error: 'Failed to serialize response', @@ -265,7 +246,6 @@ export function handleScrapeOutput( outputData = extractMultipleFormats(result.data, formats); } - if (result.receipt) outputData = { ...outputData, receipt: result.receipt }; let jsonContent: string; try { jsonContent = pretty From 433018a980c6386da330cde603f592e2f15a1d09 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:42:28 -0400 Subject: [PATCH 7/7] chore(cli): bump Alexandria beta to beta.15 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 82107a224c..3bc6f8e2de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.23.4-alexandria-beta.14", + "version": "1.23.4-alexandria-beta.15", "publishConfig": { "tag": "alexandria" },