From 84e5821385ad29e5737de3c9a972992b12112432 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 29 Aug 2026 09:04:40 -0400 Subject: [PATCH] fix: heal poisoned docs build assets --- .github/workflows/cache-purge.yml | 170 +++++++++------ scripts/cache-build-assets.mjs | 305 +++++++++++++++++++++++++++ scripts/cache-build-assets.test.ts | 158 ++++++++++++++ scripts/cache-purge-prefixes.mjs | 1 + scripts/cache-purge-prefixes.test.ts | 68 +++++- 5 files changed, 632 insertions(+), 70 deletions(-) create mode 100644 scripts/cache-build-assets.mjs create mode 100644 scripts/cache-build-assets.test.ts diff --git a/.github/workflows/cache-purge.yml b/.github/workflows/cache-purge.yml index 77e2d16fb..683cf8a86 100644 --- a/.github/workflows/cache-purge.yml +++ b/.github/workflows/cache-purge.yml @@ -2,7 +2,8 @@ name: Cloudflare Cache Purge # Purges the Cloudflare edge cache for the pages a production deploy actually # changed, so a docs edit is visible immediately instead of waiting out the 24h -# edge TTL (`SHARED_CDN_CACHE` in next.config.mjs). Nothing here changes a TTL. +# edge TTL (`SHARED_CDN_CACHE` in next.config.mjs). It also clears any cached +# error responses for current Next.js build assets. Nothing here changes a TTL. # # WHY PREFIXES, NOT URLS # The App Router serves the HTML document and the RSC flight payload at the same @@ -18,7 +19,9 @@ name: Cloudflare Cache Purge # # WHY NOT `purge_everything` # It also evicts /_next/static/**, which would put every asset on the site into -# a MISS wave on every deploy. +# a MISS wave on every deploy. This workflow instead purges the exact build +# assets referenced by current page shells and their runtime (roughly 100 URLs), +# leaving historical and unrelated static assets warm. # # KNOWN, ACCEPTED BEHAVIOURS # 1. Prefix matching is a plain string match, so `/docs/features/agents` also @@ -38,8 +41,10 @@ on: # production build and marks it `success` when the build is ready and the # production alias points at it. That is a real completion signal, it needs no # new credentials, and it arrives when the work is done rather than after a - # guessed wait. Verified present on this repo: deployments with - # `environment: Production` created by `vercel[bot]`, status `success`. + # guessed wait. Verified present on this repo: `Production` deployments whose + # successful deployment status is posted by `vercel[bot]`. The deployment + # itself is attributed to the human who initiated it, so that creator is not + # a reliable integration identifier. # # Limitations, written down rather than papered over: # - It depends on Vercel's GitHub integration staying enabled. If a deploy @@ -94,7 +99,7 @@ permissions: # its own rate limiting, so letting runs overlap is strictly safer than # serialising them and losing one. concurrency: - group: cache-purge-${{ github.event.deployment.sha || github.run_id }} + group: cache-purge-${{ github.event.deployment.id || github.run_id }} cancel-in-progress: false env: @@ -143,17 +148,15 @@ jobs: # Only successful *production* deployments from Vercel. `deployment_status` # also fires for pending/failure states and for every preview deployment. # - # The creator check is not redundant: a job-level `environment: Production` - # makes GitHub create a Production deployment too, and translate_docs.yml - # (every 30 minutes) and update-screenshots.yml both do that. Those are not - # releases. They cannot reach this trigger today, because events raised by - # GITHUB_TOKEN do not start workflow runs, but resting on that side effect - # would mean a purge on every translation sweep the day it changes. + # Check the status creator, not the deployment creator. Vercel attributes + # the deployment to the human who initiated it and posts the ready status as + # `vercel[bot]`. A job-level `environment: Production` also creates records, + # but their statuses are posted by the Actions actor and must stay excluded. if: >- github.event_name == 'workflow_dispatch' || (github.event.deployment_status.state == 'success' && github.event.deployment.environment == 'Production' && - github.event.deployment.creator.login == 'vercel[bot]') + github.event.deployment_status.creator.login == 'vercel[bot]') runs-on: ubuntu-latest steps: # Full history: the diff base is the previous deployed commit, which can be @@ -182,8 +185,6 @@ jobs: HEAD_SHA: ${{ github.event.deployment.sha }} DEPLOYMENT_ID: ${{ github.event.deployment.id }} REPO: ${{ github.repository }} - # Only deployments from the Vercel integration count as live releases. - VERCEL_CREATOR: 'vercel[bot]' PURGE_STATUS_CONTEXT: ${{ env.PURGE_STATUS_CONTEXT }} run: | set -euo pipefail @@ -243,21 +244,19 @@ jobs: echo "::error::deployment_status payload carried no numeric deployment id." exit 1 fi - # Walk production deployments newest-first and take the first one - # that succeeded on a *different* commit. Skipping same-sha entries - # means a redeploy of the current commit re-purges that commit's - # prefixes instead of computing an empty diff. + # Walk production deployments newest-first and take the first + # deployment-specific successful purge marker on a *different* + # commit. Skipping same-sha entries means a redeploy of the current + # commit re-purges that commit's prefixes instead of computing an + # empty diff. # - # Two filters keep the baseline honest: + # The purge marker proves both that the candidate was a Vercel + # success event and that its entire purge completed. This avoids + # trusting `deployment.creator`: Vercel records carry the initiating + # human there, just like unrelated Actions Production deployments. # - # creator == vercel[bot] — a job-level `environment: Production` - # also creates a Production deployment, and translate_docs.yml and - # update-screenshots.yml both do that. Those SHAs were never a live - # build. If a Vercel build for B fails while a translation run - # succeeds on B, taking B as the baseline would silently skip - # everything in A..B. - # - # id < this deployment — deployment ids increase monotonically, so + # `id < this deployment` is still required. Deployment ids increase + # monotonically, so # this is an "older than the event we are processing" test. Without # it, a deploy that finishes while this runner is queued is newer # yet still differs from head, so it would be accepted as the @@ -275,9 +274,8 @@ jobs: deployments=$(gh api \ "repos/$REPO/deployments?environment=Production&per_page=100&page=$page") [ "$(jq 'length' <<< "$deployments")" -gt 0 ] || break - ids=$(jq -r --argjson current "$DEPLOYMENT_ID" --arg creator "$VERCEL_CREATOR" ' + ids=$(jq -r --argjson current "$DEPLOYMENT_ID" ' [ .[] - | select(.creator.login == $creator) | select(.id < $current) ][] | "\(.id) \(.sha)"' <<< "$deployments") @@ -292,20 +290,21 @@ jobs: [ -n "$id" ] || continue [ -z "$base" ] || break [ "$sha" = "$head" ] && continue - # "Has it EVER succeeded", not "is its newest status success". - # GitHub appends an `inactive` status to earlier deployments in an - # environment once a newer one succeeds (auto_inactive, on by - # default). That is not happening on this repo today — the - # superseded deployment 5623426305 carries a lone `success` — but - # if it ever started, reading only the newest status would reject - # every candidate and quietly pin the workflow to broad purges. - succeeded=$(gh api "repos/$REPO/deployments/$id/statuses" \ - --jq '[.[] | select(.state == "success")] | length') - [ "${succeeded:-0}" -gt 0 ] || continue - # Keyed to this deployment, not just this commit. - purged=$(gh api "repos/$REPO/commits/$sha/statuses" \ - --jq "[.[] | select(.context == \"$PURGE_STATUS_CONTEXT/$id\" - and .state == \"success\")] | length") + # Cache commit statuses by SHA. Production deployment history is + # dominated by scheduled Actions records, often dozens on one + # commit; querying once per deployment would burn the API budget + # without learning anything new. + status_file="commit-statuses/$sha.json" + if [ ! -f "$status_file" ]; then + mkdir -p commit-statuses + gh api --paginate --slurp \ + "repos/$REPO/commits/$sha/statuses?per_page=100" > "$status_file" + fi + # Keyed to this deployment, not merely this commit. Only this + # workflow writes the marker, after every Cloudflare call wins. + purged=$(jq --arg context "$PURGE_STATUS_CONTEXT/$id" \ + '[.[][] | select(.context == $context and .state == "success")] | length' \ + "$status_file") if [ "${purged:-0}" -gt 0 ]; then base="$sha" echo "Baseline: $sha (deployment $id) — last commit with a successful purge." @@ -431,30 +430,81 @@ jobs: broad=$(jq -r '.broad' purge.json) collapsed=$(jq -r '.collapsed' purge.json) + echo "mode=$([ "$broad" = "true" ] && echo broad || echo selective)" >> "$GITHUB_OUTPUT" + echo "collapsed=$collapsed" >> "$GITHUB_OUTPUT" + + # Vercel posts success as its production alias changes. Let that alias + # propagate before reading fresh page shells or purging anything. + - name: Wait for the deploy to settle + if: ${{ github.event_name == 'deployment_status' }} + run: sleep "$SETTLE_SECONDS" + + # A new immutable asset can be requested during the alias transition and + # receive a short-lived origin 404. The zone's Cache Rule may retain that + # response much longer, independently in each Cloudflare location. Read a + # bounded set of fresh page shells plus their runtime's complete lazy-chunk + # map, then globally purge those exact current build-asset URLs; a health + # check from one runner cannot see poisoned keys in another edge location. + - name: Collect current build assets + id: assets + env: + CACHE_PROBE_TOKEN: ${{ github.run_id }}-${{ github.run_attempt }} + EVENT: ${{ github.event_name }} + run: | + set -euo pipefail + + probe_log=$(mktemp) + if node scripts/cache-build-assets.mjs > build-assets.txt 2> "$probe_log"; then + cat "$probe_log" >&2 + else + probe_status=$? + if [ "$EVENT" != "workflow_dispatch" ]; then + cat "$probe_log" >&2 + rm -f "$probe_log" + exit "$probe_status" + fi + # Manual dispatch is the escape hatch for a production/cache + # outage. Keep its already-computed broad prefixes and public asset + # targets usable even if one of the live page probes is unhealthy. + sed 's/^::error::/::warning::/' "$probe_log" >&2 + echo "::warning::Build-asset discovery failed; continuing with the manual recovery targets." + : > build-assets.txt + fi + rm -f "$probe_log" + + cat build-assets.txt >> files.txt + sort -u files.txt -o files.txt + + prefix_count=$(wc -l < prefixes.txt) + file_count=$(wc -l < files.txt) + total=$((prefix_count + file_count)) + echo "count=$total" >> "$GITHUB_OUTPUT" + { echo "### Cloudflare purge plan" echo - echo "- mode: \`$([ "$broad" = "true" ] && echo broad || echo selective)\`" - [ "$collapsed" = "true" ] && echo "- **collapsed to broad**: the diff produced more prefixes than the cap" - echo "- prefixes: $(wc -l < prefixes.txt)" - echo "- exact URLs: $(wc -l < files.txt)" + echo "- mode: \`${{ steps.compute.outputs.mode }}\`" + if [ "${{ steps.compute.outputs.collapsed }}" = "true" ]; then + echo "- **collapsed to broad**: the diff produced more prefixes than the cap" + fi + echo "- prefixes: $prefix_count" + echo "- exact URLs: $file_count" + echo "- current build assets: $(wc -l < build-assets.txt)" echo echo '```' cat prefixes.txt files.txt echo '```' } >> "$GITHUB_STEP_SUMMARY" - echo "count=$(wc -l < prefixes.txt)" >> "$GITHUB_OUTPUT" - - # Nothing mapped means the deploy touched only files that cannot change a - # rendered page (.github/**, tests, repo notes). Say so out loud; do not - # dress it up as a successful purge. + # Defensive only: build-assets.txt normally makes every production deploy + # non-empty. Keep a clear no-op result for an explicitly narrowed future + # configuration. - name: Nothing to purge - if: steps.compute.outputs.count == '0' - run: echo "::notice::No cached route changed in this deploy — no purge issued." + if: steps.assets.outputs.count == '0' + run: echo "::notice::No cached route or build asset needs purging." - name: Dry run - if: inputs.dry_run && steps.compute.outputs.count != '0' + if: inputs.dry_run && steps.assets.outputs.count != '0' run: | set -euo pipefail echo "DRY RUN — the Cloudflare API is not called. Prefixes:" @@ -467,7 +517,7 @@ jobs: # Fail loudly on missing configuration. A purge that quietly skips is the # exact failure this workflow exists to prevent. - name: Check credentials - if: ${{ !inputs.dry_run && steps.compute.outputs.count != '0' }} + if: ${{ !inputs.dry_run && steps.assets.outputs.count != '0' }} env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CF_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} @@ -482,12 +532,8 @@ jobs: exit 1 fi - - name: Wait for the deploy to settle - if: ${{ !inputs.dry_run && steps.compute.outputs.count != '0' && github.event_name == 'deployment_status' }} - run: sleep "$SETTLE_SECONDS" - - name: Purge - if: ${{ !inputs.dry_run && steps.compute.outputs.count != '0' }} + if: ${{ !inputs.dry_run && steps.assets.outputs.count != '0' }} env: CF_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CF_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} @@ -600,7 +646,7 @@ jobs: # run that fails leaves no marker, so the following run's diff widens to # include this one's range instead of stepping over it. - name: Record the purge against this commit - if: ${{ !inputs.dry_run && steps.compute.outputs.count != '0' && github.event_name == 'deployment_status' }} + if: ${{ !inputs.dry_run && steps.assets.outputs.count != '0' && github.event_name == 'deployment_status' }} env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} diff --git a/scripts/cache-build-assets.mjs b/scripts/cache-build-assets.mjs new file mode 100644 index 000000000..c68be28ad --- /dev/null +++ b/scripts/cache-build-assets.mjs @@ -0,0 +1,305 @@ +/** + * Discovers build assets referenced by the current production page shells and + * their Webpack runtime. + * + * A Vercel alias transition can briefly return a 404 for a new immutable + * `/_next/static/**` URL. Cloudflare's cache rule may retain that 404 in one + * edge location even after Vercel serves the asset normally. The purge workflow + * uses this script after the deployment settles and globally purges the exact + * current URLs, without evicting the entire static-asset namespace. + */ + +import { fileURLToPath } from 'node:url' + +export const PRODUCTION_ORIGIN = 'https://www.librechat.ai' + +// One live page per App Router page template. The stable author, blog, and +// changelog entries exercise their dynamic templates without probing every +// generated page. +export const DEFAULT_PROBE_PATHS = [ + '/', + '/about', + '/authors', + '/authors/danny', + '/blog', + '/blog/2024-04-17_blog_guide', + '/changelog', + '/changelog/config_v1.0.0', + '/cookie', + '/de', + '/demo/privacy', + '/demo/terms', + '/docs', + '/de/docs', + '/privacy', + '/subscribe', + '/toolkit', + '/toolkit/creds-generator', + '/toolkit/yaml-checker', + '/tos', + '/unsubscribe', +] + +const CACHE_PROBE_PARAM = '__librechat_cache_probe' +const STATIC_PREFIX = '/_next/static/' +const WEBPACK_RUNTIME_PATH = /\/static\/chunks\/webpack-[^/]+\.js$/u +const REQUEST_TIMEOUT_MS = 15_000 +const MAX_CONCURRENCY = 8 + +function decodeAttribute(value) { + return value.replaceAll('&', '&').replaceAll('&', '&').replaceAll('&', '&') +} + +/** Extract same-origin Next.js build assets from script and link tags. */ +export function extractBuildAssetUrls(html, origin = PRODUCTION_ORIGIN) { + const site = new URL(origin) + const urls = new Set() + const tags = html.matchAll( + /<(?:script|link)\b[^>]*?\b(?:src|href)\s*=\s*(?:"([^"]+)"|'([^']+)')[^>]*>/giu, + ) + + for (const match of tags) { + const value = decodeAttribute(match[1] ?? match[2]) + let url + try { + url = new URL(value, site) + } catch { + continue + } + if (url.origin !== site.origin || !url.pathname.startsWith(STATIC_PREFIX)) continue + url.hash = '' + urls.add(url.href) + } + + return [...urls].sort() +} + +function runtimeAssetUrl(value, origin) { + const url = new URL(value, new URL('/_next/', origin)) + if (url.origin !== new URL(origin).origin || !url.pathname.startsWith(STATIC_PREFIX)) return null + if (url.pathname.endsWith('/')) return null + url.hash = '' + return url.href +} + +function parseNumericStringMap(source) { + const entries = [...source.matchAll(/"?(\d+)"?\s*:\s*"([A-Za-z0-9_-]+)"/gu)].map((match) => [ + match[1], + match[2], + ]) + return new Map(entries) +} + +function materializeMappedAssets(factory, parameter, directory, extension, origin) { + const maps = [...factory.matchAll(/\{((?:\s*"?\d+"?\s*:\s*"[A-Za-z0-9_-]+"\s*,?\s*)+)\}/gu)].map( + (match) => parseNumericStringMap(match[1]), + ) + if (maps.length === 0) return [] + + if (maps.length === 1) { + const escapedParameter = parameter.replaceAll(/[.*+?^${}()|[\]\\]/gu, '\\$&') + const prependsChunkId = new RegExp( + `\\+\\s*${escapedParameter}\\s*\\+\\s*["']\\.["']\\s*\\+`, + 'u', + ).test(factory) + return [...maps[0]].flatMap(([id, value]) => { + // Some factories use `id + "." + hash`; others map the id directly to + // the complete filename stem. Follow the expression the browser uses. + const stem = prependsChunkId ? `${id}.${value}` : value + const url = runtimeAssetUrl(`${directory}/${stem}.${extension}`, origin) + return url ? [url] : [] + }) + } + + // The hash map is the largest map and appears last on a tie. Earlier maps + // optionally replace a numeric chunk id with a content-derived basename. + let hashMapIndex = 0 + for (let index = 1; index < maps.length; index += 1) { + if (maps[index].size >= maps[hashMapIndex].size) hashMapIndex = index + } + const hashMap = maps[hashMapIndex] + const basenameMaps = maps.slice(0, hashMapIndex) + + const urls = [] + for (const [id, hash] of hashMap) { + let basename = id + for (const map of basenameMaps) basename = map.get(id) ?? basename + const url = runtimeAssetUrl(`${directory}/${basename}.${hash}.${extension}`, origin) + if (url) urls.push(url) + } + return urls +} + +/** + * Extract lazy JS/CSS assets from Next.js' Webpack runtime without executing + * remote code. Direct filenames are string literals; ordinary lazy JS chunks + * are represented by numeric-id -> filename/hash maps inside `webpackRequire.u`. + */ +export function extractWebpackRuntimeAssetUrls(source, origin = PRODUCTION_ORIGIN) { + const urls = new Set() + + for (const match of source.matchAll(/["'](static\/(?:chunks|css|media)\/[^"'\\\s]*)["']/gu)) { + const url = runtimeAssetUrl(match[1], origin) + if (url) urls.add(url) + } + + const chunkFactory = source.match( + /\.u=([A-Za-z_$][\w$]*)=>([\s\S]*?),\s*[A-Za-z_$][\w$]*\.miniCssF=/u, + ) + if (!chunkFactory) { + throw new Error('Could not find the Webpack lazy-chunk factory in the production runtime') + } + + for (const url of materializeMappedAssets( + chunkFactory[2], + chunkFactory[1], + 'static/chunks', + 'js', + origin, + )) { + urls.add(url) + } + + const cssFactory = source.match( + /\.miniCssF=([A-Za-z_$][\w$]*)=>([\s\S]*?),\s*[A-Za-z_$][\w$]*\.g=/u, + ) + if (!cssFactory) { + throw new Error('Could not find the Webpack lazy-CSS factory in the production runtime') + } + for (const url of materializeMappedAssets( + cssFactory[2], + cssFactory[1], + 'static/css', + 'css', + origin, + )) { + urls.add(url) + } + + if (urls.size === 0) { + throw new Error('No lazy build assets were found in the production Webpack runtime') + } + return [...urls].sort() +} + +/** Add a unique query key without disturbing any existing query parameters. */ +export function withCacheProbe(url, token) { + const probed = new URL(url) + probed.searchParams.set(CACHE_PROBE_PARAM, token) + return probed.href +} + +async function fetchWithRetry(fetchImpl, url, init) { + let lastError + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + const response = await fetchImpl(url, { + ...init, + redirect: 'follow', + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + headers: { + 'user-agent': 'LibreChat-docs-cache-health/1.0', + ...init?.headers, + }, + }) + if (response.status < 500 || attempt === 3) return response + lastError = new Error(`${url} returned HTTP ${response.status}`) + } catch (error) { + lastError = error + } + + if (attempt < 3) { + await new Promise((resolve) => setTimeout(resolve, attempt * 250)) + } + } + throw lastError +} + +async function mapConcurrent(items, worker) { + const results = new Array(items.length) + let cursor = 0 + + async function run() { + while (cursor < items.length) { + const index = cursor + cursor += 1 + results[index] = await worker(items[index], index) + } + } + + const workers = Array.from({ length: Math.min(MAX_CONCURRENCY, items.length) }, () => run()) + await Promise.all(workers) + return results +} + +export async function discoverCurrentBuildAssets({ + fetchImpl = fetch, + origin = PRODUCTION_ORIGIN, + probePaths = DEFAULT_PROBE_PATHS, + token = `${Date.now()}`, +} = {}) { + const pageResponses = await mapConcurrent(probePaths, async (path, index) => { + const pageUrl = new URL(path, origin) + const response = await fetchWithRetry( + fetchImpl, + withCacheProbe(pageUrl, `${token}-page-${index}`), + { method: 'GET' }, + ) + if (!response.ok) { + throw new Error(`Fresh page probe failed for ${pageUrl.href}: HTTP ${response.status}`) + } + return response.text() + }) + + const shellAssets = [ + ...new Set(pageResponses.flatMap((html) => extractBuildAssetUrls(html, origin))), + ].sort() + if (shellAssets.length === 0) { + throw new Error('No Next.js build assets were found in the production page shells') + } + + const runtimes = shellAssets.filter((asset) => WEBPACK_RUNTIME_PATH.test(new URL(asset).pathname)) + if (runtimes.length === 0) { + throw new Error('No Webpack runtime was found in the production page shells') + } + + const runtimeResponses = await mapConcurrent(runtimes, async (runtime, index) => { + const response = await fetchWithRetry( + fetchImpl, + withCacheProbe(runtime, `${token}-runtime-${index}`), + { method: 'GET' }, + ) + if (!response.ok) { + throw new Error(`Fresh Webpack runtime probe failed for ${runtime}: HTTP ${response.status}`) + } + return response.text() + }) + + const runtimeAssets = runtimeResponses.flatMap((source) => + extractWebpackRuntimeAssetUrls(source, origin), + ) + const assets = [...new Set([...shellAssets, ...runtimeAssets])].sort() + return assets +} + +async function main() { + const token = process.env.CACHE_PROBE_TOKEN || `${Date.now()}-${process.pid}` + const probePaths = process.env.CACHE_PROBE_PATHS + ? process.env.CACHE_PROBE_PATHS.split(',') + .map((path) => path.trim()) + .filter(Boolean) + : DEFAULT_PROBE_PATHS + const assets = await discoverCurrentBuildAssets({ probePaths, token }) + + process.stderr.write( + `Collected ${assets.length} current build assets from ${probePaths.length} fresh page shells and their Webpack runtime.\n`, + ) + process.stdout.write(`${assets.join('\n')}\n`) +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((error) => { + process.stderr.write(`::error::${error.message}\n`) + process.exitCode = 1 + }) +} diff --git a/scripts/cache-build-assets.test.ts b/scripts/cache-build-assets.test.ts new file mode 100644 index 000000000..81ea1caee --- /dev/null +++ b/scripts/cache-build-assets.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, it, vi } from 'vitest' +import { + DEFAULT_PROBE_PATHS, + discoverCurrentBuildAssets, + extractBuildAssetUrls, + extractWebpackRuntimeAssetUrls, + withCacheProbe, +} from './cache-build-assets.mjs' + +const origin = 'https://www.librechat.ai' +const toUrl = (input: string | URL | Request) => + new URL(input instanceof Request ? input.url : input) + +describe('DEFAULT_PROBE_PATHS', () => { + it('covers each static and dynamic page template with a live route', () => { + expect(DEFAULT_PROBE_PATHS).toEqual([ + '/', + '/about', + '/authors', + '/authors/danny', + '/blog', + '/blog/2024-04-17_blog_guide', + '/changelog', + '/changelog/config_v1.0.0', + '/cookie', + '/de', + '/demo/privacy', + '/demo/terms', + '/docs', + '/de/docs', + '/privacy', + '/subscribe', + '/toolkit', + '/toolkit/creds-generator', + '/toolkit/yaml-checker', + '/tos', + '/unsubscribe', + ]) + }) +}) + +describe('extractBuildAssetUrls', () => { + it('collects and deduplicates same-origin Next.js assets from script and link tags', () => { + const html = ` + + + + + + + + ` + + expect(extractBuildAssetUrls(html, origin)).toEqual([ + 'https://www.librechat.ai/_next/static/chunks/webpack-123.js', + 'https://www.librechat.ai/_next/static/css/app.css?x=1&y=2', + ]) + }) +}) + +describe('withCacheProbe', () => { + it('preserves an existing query while replacing the probe key', () => { + const url = withCacheProbe(`${origin}/_next/static/a.js?x=1`, 'run 2') + expect(url).toBe(`${origin}/_next/static/a.js?x=1&__librechat_cache_probe=run+2`) + }) +}) + +describe('extractWebpackRuntimeAssetUrls', () => { + it('collects direct assets and materializes mapped lazy chunks without evaluating code', () => { + const runtime = ` + r.u=e=>7===e + ? "static/chunks/special.js" + : "static/chunks/"+(({12:"named"})[e]||e)+"."+({12:"abc123",34:"def456"})[e]+".js", + r.miniCssF=e=>9===e + ? "static/css/special.css" + : "static/css/"+(({12:"theme"})[e]||e)+"."+({12:"csshash",56:"othercss"})[e]+".css", + r.g={} + ` + + expect(extractWebpackRuntimeAssetUrls(runtime, origin)).toEqual([ + `${origin}/_next/static/chunks/34.def456.js`, + `${origin}/_next/static/chunks/named.abc123.js`, + `${origin}/_next/static/chunks/special.js`, + `${origin}/_next/static/css/56.othercss.css`, + `${origin}/_next/static/css/special.css`, + `${origin}/_next/static/css/theme.csshash.css`, + ]) + }) + + it('fails closed when the runtime format cannot be understood', () => { + expect(() => extractWebpackRuntimeAssetUrls('not a webpack runtime', origin)).toThrow( + 'Could not find the Webpack lazy-chunk factory', + ) + }) + + it('uses a direct miniCssF map value as the complete filename stem', () => { + const runtime = ` + r.u=e=>"static/chunks/"+e+"."+({42:"lazyhash"})[e]+".js", + r.miniCssF=e=>"static/css/"+({12:"contenthash"})[e]+".css", + r.g={} + ` + + expect(extractWebpackRuntimeAssetUrls(runtime, origin)).toEqual([ + `${origin}/_next/static/chunks/42.lazyhash.js`, + `${origin}/_next/static/css/contenthash.css`, + ]) + }) +}) + +describe('discoverCurrentBuildAssets', () => { + it('collects the union of assets from fresh production page shells', async () => { + const fetchImpl = vi.fn(async (input: string | URL | Request) => { + const { pathname } = toUrl(input) + if (pathname.endsWith('/webpack-test.js')) { + return new Response( + 'r.u=e=>"static/chunks/"+e+"."+({42:"lazyhash"})[e]+".js",' + + 'r.miniCssF=e=>"static/css/lazy.css",r.g={}', + ) + } + const chunk = pathname === '/' ? 'shared' : 'docs' + return new Response( + `` + + '' + + '', + ) + }) + + await expect( + discoverCurrentBuildAssets({ + fetchImpl, + origin, + probePaths: ['/', '/docs'], + token: 'test', + }), + ).resolves.toEqual([ + `${origin}/_next/static/chunks/42.lazyhash.js`, + `${origin}/_next/static/chunks/docs.js`, + `${origin}/_next/static/chunks/shared.js`, + `${origin}/_next/static/chunks/webpack-test.js`, + `${origin}/_next/static/css/lazy.css`, + ]) + expect(fetchImpl).toHaveBeenCalledTimes(3) + }) + + it('fails when a fresh production page shell cannot be fetched', async () => { + const fetchImpl = vi.fn(async () => new Response(null, { status: 404 })) + await expect( + discoverCurrentBuildAssets({ fetchImpl, origin, probePaths: ['/'], token: 'test' }), + ).rejects.toThrow('Fresh page probe failed') + }) + + it('fails loudly when no build assets can be discovered', async () => { + const fetchImpl = vi.fn(async () => new Response('
No scripts
')) + await expect( + discoverCurrentBuildAssets({ fetchImpl, origin, probePaths: ['/'], token: 'test' }), + ).rejects.toThrow('No Next.js build assets') + }) +}) diff --git a/scripts/cache-purge-prefixes.mjs b/scripts/cache-purge-prefixes.mjs index 349ff4d20..162fa08dc 100644 --- a/scripts/cache-purge-prefixes.mjs +++ b/scripts/cache-purge-prefixes.mjs @@ -94,6 +94,7 @@ const INERT = [ /^e2e\//, /^__tests__\//, /^scripts\/screenshots\//, + /^scripts\/cache-(build-assets|purge-prefixes)\.mjs$/, /(^|\/)__tests__\//, /\.test\.(ts|tsx|mjs|js)$/, /^(README|LICENSE|CONTRIBUTING|SECURITY|CHANGELOG|PURGE-NOTES)(\.md)?$/, diff --git a/scripts/cache-purge-prefixes.test.ts b/scripts/cache-purge-prefixes.test.ts index bb732300f..00bad2aba 100644 --- a/scripts/cache-purge-prefixes.test.ts +++ b/scripts/cache-purge-prefixes.test.ts @@ -172,6 +172,55 @@ describe('the workflow feeds the mapper what it needs', () => { expect(workflow).toMatch(/git diff --name-status -z --no-renames/) }) + /** + * Vercel attributes a deployment to the human who initiated it, then posts + * the ready status as vercel[bot]. Checking deployment.creator made every + * automatic purge run skip before it reached a step. + */ + it('identifies Vercel by the deployment-status creator', () => { + expect(workflow).toContain("github.event.deployment_status.creator.login == 'vercel[bot]'") + expect(workflow).not.toContain("github.event.deployment.creator.login == 'vercel[bot]'") + }) + + /** + * Vercel can redeploy the same SHA. Grouping by SHA would let GitHub discard + * one of those pending purge runs, even though each deployment needs its own + * baseline and marker. + */ + it('keys concurrency by deployment id so same-sha redeployments are distinct', () => { + expect(workflow).toContain( + 'group: cache-purge-${{ github.event.deployment.id || github.run_id }}', + ) + expect(workflow).not.toContain( + 'group: cache-purge-${{ github.event.deployment.sha || github.run_id }}', + ) + }) + + it('adds current build assets before deciding there is nothing to purge', () => { + expect(workflow).toContain('node scripts/cache-build-assets.mjs > build-assets.txt') + expect(workflow).toContain("if: steps.assets.outputs.count == '0'") + expect(workflow).not.toContain('if: steps.compute.outputs.count') + }) + + /** + * An operator uses workflow_dispatch to recover an unhealthy production + * site. A failed live probe must not prevent the broad page/public targets + * computed earlier in the job from reaching Cloudflare. + */ + it('keeps manual recovery targets when build-asset discovery fails', () => { + expect(workflow).toContain('if [ "$EVENT" != "workflow_dispatch" ]; then') + expect(workflow).toContain(': > build-assets.txt') + expect(workflow).toContain( + 'Build-asset discovery failed; continuing with the manual recovery targets.', + ) + }) + + it('accepts a baseline only when its deployment-specific purge marker exists', () => { + expect(workflow).toContain('PURGE_STATUS_CONTEXT/$id') + expect(workflow).toContain('select(.context == $context and .state == "success")') + expect(workflow).not.toContain('select(.creator.login == $creator)') + }) + /** * A peer that accepts the connection and then stops responding would otherwise * never return, so the retry loop never sees a failure and the job sits until @@ -667,14 +716,17 @@ describe('fallbacks', () => { expect(prefixes).toContain('www.librechat.ai/docs/features') }) - it.each(['.github/workflows/ci.yml', 'e2e/home.spec.ts', 'README.md', 'lib/i18n/tm.test.ts'])( - 'purges nothing for %s', - (file) => { - const result = prefixesForFile(file, locales) - expect(result.broad).toBe(false) - expect(result.prefixes).toEqual([]) - }, - ) + it.each([ + '.github/workflows/ci.yml', + 'e2e/home.spec.ts', + 'README.md', + 'lib/i18n/tm.test.ts', + 'scripts/cache-build-assets.mjs', + ])('purges nothing for %s', (file) => { + const result = prefixesForFile(file, locales) + expect(result.broad).toBe(false) + expect(result.prefixes).toEqual([]) + }) }) describe('computePurge', () => {