From a563de5897f83d4140beccece7761ead7658560a Mon Sep 17 00:00:00 2001 From: Julian Gruber Date: Thu, 1 Oct 2026 11:59:37 +0200 Subject: [PATCH] feat(firewall): download from a random origin with cross-origin fallback The free-edition binary is now downloadable from two origins: GitHub release assets, and the Socket-owned mirror at install.socket.dev/firewall/dl// (served by firewall-download-server in depscan). firewallDownloadUrls returns the equivalent origins in a fixed order; downloadFirewall tries them in a Fisher-Yates permutation from shuffledIndexes, so roughly half the fleet's downloads keep the mirror's edge cache warm, which is what lets it keep serving pinned binaries during a GitHub release-asset incident. Adding a third origin needs no change to the selection. downloadToolWithRetry takes the origin list and gives each origin the whole 30s/60s retry schedule: a round tries every origin still in play, and only a round with no success sits out the next delay. An origin that fails with an error a retry cannot change (a 404, a 403) drops out of later rounds, so a mirror that lacks the asset cannot use up the retries GitHub needs to ride out a 504. The error rethrown is the last transient one when there was one, since that is the outage worth reporting; otherwise the last error seen. The checksum table in this action's source validates every download regardless of origin, so the second host cannot alter what gets installed. Enterprise stays GitHub-only: firewall-release is private and not mirrored. test-sfw-mirror.yml runs on every pull request and forces the failure this guards against: GitHub or the mirror pointed at 127.0.0.1 in the hosts file, with the install expected to succeed from the other origin on windows-2025 and ubuntu-26.04. --- .github/workflows/test-sfw-mirror.yml | 74 +++++++++++++ dist/main.js | 91 +++++++++++++--- src/tools/firewall.js | 132 +++++++++++++++++----- test/unit/tools/firewall.test.mts | 151 ++++++++++++++++++++++++-- 4 files changed, 397 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/test-sfw-mirror.yml diff --git a/.github/workflows/test-sfw-mirror.yml b/.github/workflows/test-sfw-mirror.yml new file mode 100644 index 0000000..aaa0f7c --- /dev/null +++ b/.github/workflows/test-sfw-mirror.yml @@ -0,0 +1,74 @@ +name: 'test: sfw mirror' +run-name: 'test: sfw mirror' + +# Sfw binary download origins can fail, ensure the action still works +# if only one of them is unavailable + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + fault-download-origin: + name: 'Block a download origin (${{ matrix.os }}, ${{ matrix.blocked }})' + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [windows-2025, ubuntu-26.04] + # Each origin must carry the install alone. The github case only + # passes with the mirror fallback in the checked-out action. + blocked: [github, mirror] + steps: + - name: 'Bootstrap checkout' + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + SERVER_URL: ${{ github.server_url }} + REPOSITORY: ${{ github.repository }} + TRIGGER_REF: ${{ github.sha }} + run: | + set -euo pipefail + git init -q + git config --local advice.detachedHead false + git remote add origin "${SERVER_URL}/${REPOSITORY}" + AUTH_B64="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 | tr -d '\n')" + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0="http.${SERVER_URL}/.extraheader" + export GIT_CONFIG_VALUE_0="AUTHORIZATION: basic ${AUTH_B64}" + git fetch --no-tags --prune --depth 1 origin "${TRIGGER_REF}" + git checkout -q --detach FETCH_HEAD + - name: 'Block the origin in the hosts file' + shell: bash + env: + BLOCKED: ${{ matrix.blocked }} + RUNNER_OS: ${{ runner.os }} + run: | + set -euo pipefail + if [ "$RUNNER_OS" = Windows ]; then HOSTS="$WINDIR/System32/drivers/etc/hosts"; else HOSTS=/etc/hosts; fi + if [ "$BLOCKED" = github ]; then + hosts="github.com objects.githubusercontent.com release-assets.githubusercontent.com" + else + hosts="install.socket.dev" + fi + # 127.0.0.1 refuses the connection at once. A black-hole address would + # make every attempt wait out a TCP connect timeout, which on Linux + # outlives the job. + for h in $hosts; do printf '127.0.0.1 %s\n' "$h" | sudo tee -a "$HOSTS" > /dev/null 2>&1 || printf '127.0.0.1 %s\n' "$h" >> "$HOSTS"; done + [ "$RUNNER_OS" = Windows ] && ipconfig //flushdns > /dev/null || true + - name: 'Install socket firewall via the remaining origin' + uses: ./ + with: + mode: firewall + job-summary: errors + use-cache: 'false' + - name: 'Run the installed binary' + shell: bash + run: sfw --version diff --git a/dist/main.js b/dist/main.js index f21caa0..e4bb2bf 100644 --- a/dist/main.js +++ b/dist/main.js @@ -22135,6 +22135,12 @@ const FIREWALL_CHECKSUMS = { */ const DOWNLOAD_RETRY_DELAYS_SECONDS = [30, 60]; /** +* Socket-owned mirror of the sfw-free release binaries, relayed by +* firewall-download-server in depscan. Free edition only: the enterprise +* repository is private and not mirrored. +*/ +const FIREWALL_FREE_MIRROR_BASE_URL = "https://install.socket.dev/firewall/dl"; +/** * Name the firewall binary is cached and executed under. */ const FIREWALL_EXEC_NAME = "sfw"; @@ -22163,14 +22169,15 @@ async function downloadFirewall({ edition = "free", ...inputs }) { versionToDownload, process.arch ]; - const url = `https://github.com/SocketDev/${repo}/releases/download/${versionToDownload}/${nameDownload}`; + const origins = firewallDownloadUrls(edition, repo, versionToDownload, nameDownload); + const urls = shuffledIndexes(origins.length).map((i) => origins[i]); let pathCache; if (inputs.useCache) pathCache = find(...cacheOptions); if (!pathCache) { - debug(`downloading Socket Firewall binary from: ${url}`); + debug(`downloading Socket Firewall binary from: ${urls.join(", ")}`); let pathDownload; try { - pathDownload = await downloadToolWithRetry(url); + pathDownload = await downloadToolWithRetry(urls); } catch (error) { throw new Error(`Failed to download Socket Firewall binary: ${(0, import_message.errorMessage)(error)}`); } @@ -22196,25 +22203,60 @@ async function downloadFirewall({ edition = "free", ...inputs }) { } } /** -* `downloadTool` with attempts layered on top of its own. The last error is -* rethrown untouched so the caller still reports the real cause. +* `downloadTool` with attempts layered on top of its own, across equivalent +* origins. Every origin gets the whole delay table to itself: a round tries +* each origin still in play, and a round that leaves none of them succeeding +* sits out the next delay before going again. An origin that fails with an +* error a retry cannot change (a 404) drops out of later rounds, so a mirror +* that lacks the asset cannot use up the retries GitHub needs to ride out a +* 504. The error rethrown is the last transient one when there was one, since +* that is the outage worth reporting; otherwise the last error seen. * -* @param {string} url Asset to download. +* @param {string[]} urls Equivalent origins for the same asset, in the order +* to try them. * * @returns {Promise} Path the asset was downloaded to. */ -async function downloadToolWithRetry(url) { +async function downloadToolWithRetry(urls) { + let alive = urls; let lastError; - for (let attempt = 0; attempt <= DOWNLOAD_RETRY_DELAYS_SECONDS.length; attempt += 1) try { - return await downloadTool(url); - } catch (error) { - lastError = error; - const seconds = DOWNLOAD_RETRY_DELAYS_SECONDS[attempt]; - if (seconds === void 0 || !isRetryableDownloadError(error)) break; - warning(`Socket Firewall binary download failed (attempt ${attempt + 1} of ${DOWNLOAD_RETRY_DELAYS_SECONDS.length + 1}): ${(0, import_message.errorMessage)(error)}. Retrying in ${seconds}s.`); + let lastRetryableError; + for (let round = 0;; round += 1) { + const stillAlive = []; + for (const url of alive) try { + return await downloadTool(url); + } catch (error) { + lastError = error; + if (isRetryableDownloadError(error)) { + lastRetryableError = error; + stillAlive.push(url); + warning(`Socket Firewall binary download from ${url} failed (attempt ${round + 1} of ${DOWNLOAD_RETRY_DELAYS_SECONDS.length + 1}): ${(0, import_message.errorMessage)(error)}.`); + } else warning(`Socket Firewall binary download from ${url} failed: ${(0, import_message.errorMessage)(error)}. Not retrying this origin.`); + } + const seconds = DOWNLOAD_RETRY_DELAYS_SECONDS[round]; + if (stillAlive.length === 0 || seconds === void 0) throw lastRetryableError ?? lastError; + warning(`Retrying ${stillAlive.join(", ")} in ${seconds}s.`); await setTimeout$1(seconds * 1e3); + alive = stillAlive; } - throw lastError; +} +/** +* Equivalent origins to download one release asset from. GitHub release +* assets come first; the free edition is also mirrored on Socket-owned +* infrastructure. The enterprise repository is private and not mirrored, so it +* stays GitHub-only. Callers decide the order to try them in. +* +* @param {string} edition Firewall edition being installed. +* @param {string} repo GitHub repository the release lives in. +* @param {string} version Release tag to download. +* @param {string} asset Release asset name. +* +* @returns {string[]} Download URLs, GitHub first. +*/ +function firewallDownloadUrls(edition, repo, version, asset) { + const urls = [`https://github.com/SocketDev/${repo}/releases/download/${version}/${asset}`]; + if (edition === "free") urls.push(`${FIREWALL_FREE_MIRROR_BASE_URL}/${version}/${asset}`); + return urls; } function firewallReleaseVersion(requestedVersion) { let versionToDownload = FIREWALL_VERSION; @@ -22254,6 +22296,25 @@ function isRetryableDownloadError(error) { return status >= 500 || status === 408 || status === 429; } /** +* A random permutation of `0 .. length - 1` (Fisher-Yates), for callers that +* need to try equivalent options in an unbiased order. +* +* @param {number} length How many indexes to permute. +* @param {() => number} random Injectable for tests. +* +* @returns {number[]} Every index once, in random order. +*/ +function shuffledIndexes(length, random = Math.random) { + const order = Array.from({ length }, (_, i) => i); + for (let i = order.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)); + const swap = order[i]; + order[i] = order[j]; + order[j] = swap; + } + return order; +} +/** * Compare a downloaded binary against the hash pinned for its release and * throw when they differ. Both hashes are printed so an operator can tell a * stale pin apart from a tampered download. diff --git a/src/tools/firewall.js b/src/tools/firewall.js index dd6dcb0..51c7eaf 100644 --- a/src/tools/firewall.js +++ b/src/tools/firewall.js @@ -87,6 +87,14 @@ export const FIREWALL_CHECKSUMS = { */ export const DOWNLOAD_RETRY_DELAYS_SECONDS = [30, 60] +/** + * Socket-owned mirror of the sfw-free release binaries, relayed by + * firewall-download-server in depscan. Free edition only: the enterprise + * repository is private and not mirrored. + */ +export const FIREWALL_FREE_MIRROR_BASE_URL = + 'https://install.socket.dev/firewall/dl' + /** * Name the firewall binary is cached and executed under. */ @@ -147,8 +155,16 @@ export async function downloadFirewall({ edition = 'free', ...inputs }) { process.arch, ] - // construct the download url - const url = `https://github.com/SocketDev/${repo}/releases/download/${versionToDownload}/${nameDownload}` + // construct the download urls, tried in a random order per job: half the + // fleet's downloads keep the mirror's edge cache warm, which is what lets it + // keep serving during a GitHub release-asset incident. + const origins = firewallDownloadUrls( + edition, + repo, + versionToDownload, + nameDownload, + ) + const urls = shuffledIndexes(origins.length).map(i => origins[i]) let pathCache @@ -159,13 +175,13 @@ export async function downloadFirewall({ edition = 'free', ...inputs }) { // no cache, download new if (!pathCache) { - debug(`downloading Socket Firewall binary from: ${url}`) + debug(`downloading Socket Firewall binary from: ${urls.join(', ')}`) let pathDownload try { // download it - pathDownload = await downloadToolWithRetry(url) + pathDownload = await downloadToolWithRetry(urls) } catch (error) { throw new Error( `Failed to download Socket Firewall binary: ${errorMessage(error)}`, @@ -228,41 +244,83 @@ export async function downloadFirewall({ edition = 'free', ...inputs }) { } /** - * `downloadTool` with attempts layered on top of its own. The last error is - * rethrown untouched so the caller still reports the real cause. + * `downloadTool` with attempts layered on top of its own, across equivalent + * origins. Every origin gets the whole delay table to itself: a round tries + * each origin still in play, and a round that leaves none of them succeeding + * sits out the next delay before going again. An origin that fails with an + * error a retry cannot change (a 404) drops out of later rounds, so a mirror + * that lacks the asset cannot use up the retries GitHub needs to ride out a + * 504. The error rethrown is the last transient one when there was one, since + * that is the outage worth reporting; otherwise the last error seen. * - * @param {string} url Asset to download. + * @param {string[]} urls Equivalent origins for the same asset, in the order + * to try them. * * @returns {Promise} Path the asset was downloaded to. */ -export async function downloadToolWithRetry(url) { +export async function downloadToolWithRetry(urls) { + let alive = urls let lastError + let lastRetryableError + + for (let round = 0; ; round += 1) { + const stillAlive = [] + + for (const url of alive) { + try { + return await downloadTool(url) + } catch (error) { + lastError = error + + if (isRetryableDownloadError(error)) { + lastRetryableError = error + stillAlive.push(url) + warning( + `Socket Firewall binary download from ${url} failed (attempt ${round + 1} of ${DOWNLOAD_RETRY_DELAYS_SECONDS.length + 1}): ${errorMessage(error)}.`, + ) + } else { + warning( + `Socket Firewall binary download from ${url} failed: ${errorMessage(error)}. Not retrying this origin.`, + ) + } + } + } - for ( - let attempt = 0; - attempt <= DOWNLOAD_RETRY_DELAYS_SECONDS.length; - attempt += 1 - ) { - try { - return await downloadTool(url) - } catch (error) { - lastError = error + const seconds = DOWNLOAD_RETRY_DELAYS_SECONDS[round] - const seconds = DOWNLOAD_RETRY_DELAYS_SECONDS[attempt] + if (stillAlive.length === 0 || seconds === undefined) { + throw lastRetryableError ?? lastError + } - if (seconds === undefined || !isRetryableDownloadError(error)) { - break - } + warning(`Retrying ${stillAlive.join(', ')} in ${seconds}s.`) + await setTimeout(seconds * 1000) + alive = stillAlive + } +} - warning( - `Socket Firewall binary download failed (attempt ${attempt + 1} of ${DOWNLOAD_RETRY_DELAYS_SECONDS.length + 1}): ${errorMessage(error)}. Retrying in ${seconds}s.`, - ) +/** + * Equivalent origins to download one release asset from. GitHub release + * assets come first; the free edition is also mirrored on Socket-owned + * infrastructure. The enterprise repository is private and not mirrored, so it + * stays GitHub-only. Callers decide the order to try them in. + * + * @param {string} edition Firewall edition being installed. + * @param {string} repo GitHub repository the release lives in. + * @param {string} version Release tag to download. + * @param {string} asset Release asset name. + * + * @returns {string[]} Download URLs, GitHub first. + */ +export function firewallDownloadUrls(edition, repo, version, asset) { + const urls = [ + `https://github.com/SocketDev/${repo}/releases/download/${version}/${asset}`, + ] - await setTimeout(seconds * 1000) - } + if (edition === 'free') { + urls.push(`${FIREWALL_FREE_MIRROR_BASE_URL}/${version}/${asset}`) } - throw lastError + return urls } export function firewallReleaseVersion(requestedVersion) { @@ -315,6 +373,26 @@ export function isRetryableDownloadError(error) { return status >= 500 || status === 408 || status === 429 } +/** + * A random permutation of `0 .. length - 1` (Fisher-Yates), for callers that + * need to try equivalent options in an unbiased order. + * + * @param {number} length How many indexes to permute. + * @param {() => number} random Injectable for tests. + * + * @returns {number[]} Every index once, in random order. + */ +export function shuffledIndexes(length, random = Math.random) { + const order = Array.from({ length }, (_, i) => i) + for (let i = order.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)) + const swap = order[i] + order[i] = order[j] + order[j] = swap + } + return order +} + /** * Compare a downloaded binary against the hash pinned for its release and * throw when they differ. Both hashes are printed so an operator can tell a diff --git a/test/unit/tools/firewall.test.mts b/test/unit/tools/firewall.test.mts index abdcce7..6ce3a86 100644 --- a/test/unit/tools/firewall.test.mts +++ b/test/unit/tools/firewall.test.mts @@ -13,7 +13,9 @@ import { downloadToolWithRetry, FIREWALL_DISTRIBUTIONS, FIREWALL_EXEC_NAME, + firewallDownloadUrls, isRetryableDownloadError, + shuffledIndexes, } from '../../../src/tools/firewall.js' const { mockDownloadTool, sleepDelays } = vi.hoisted(() => ({ @@ -147,14 +149,15 @@ describe('isRetryableDownloadError', () => { }) describe('downloadToolWithRetry', () => { + const GITHUB = 'https://github.test/sfw' + const MIRROR = 'https://mirror.test/sfw' + it('returns the path once an attempt succeeds', async () => { mockDownloadTool .mockRejectedValueOnce(httpError(504)) .mockResolvedValueOnce('/tmp/sfw') - await expect( - downloadToolWithRetry('https://example.test/sfw'), - ).resolves.toBe('/tmp/sfw') + await expect(downloadToolWithRetry([GITHUB])).resolves.toBe('/tmp/sfw') expect(mockDownloadTool).toHaveBeenCalledTimes(2) expect(sleepDelays).toEqual([30_000]) }) @@ -162,9 +165,9 @@ describe('downloadToolWithRetry', () => { it('rethrows the last error after exhausting its attempts', async () => { mockDownloadTool.mockRejectedValue(httpError(504)) - await expect( - downloadToolWithRetry('https://example.test/sfw'), - ).rejects.toThrow('Unexpected HTTP response: 504') + await expect(downloadToolWithRetry([GITHUB])).rejects.toThrow( + 'Unexpected HTTP response: 504', + ) expect(mockDownloadTool).toHaveBeenCalledTimes(3) expect(sleepDelays).toEqual([30_000, 60_000]) }) @@ -172,10 +175,140 @@ describe('downloadToolWithRetry', () => { it('does not spend attempts on a missing asset', async () => { mockDownloadTool.mockRejectedValue(httpError(404)) - await expect( - downloadToolWithRetry('https://example.test/sfw'), - ).rejects.toThrow('Unexpected HTTP response: 404') + await expect(downloadToolWithRetry([GITHUB])).rejects.toThrow( + 'Unexpected HTTP response: 404', + ) expect(mockDownloadTool).toHaveBeenCalledTimes(1) expect(sleepDelays).toEqual([]) }) + + it('gives every origin the whole retry schedule', async () => { + mockDownloadTool.mockRejectedValue(httpError(504)) + + await expect(downloadToolWithRetry([MIRROR, GITHUB])).rejects.toThrow( + 'Unexpected HTTP response: 504', + ) + expect(mockDownloadTool.mock.calls).toEqual([ + [MIRROR], + [GITHUB], + [MIRROR], + [GITHUB], + [MIRROR], + [GITHUB], + ]) + expect(sleepDelays).toEqual([30_000, 60_000]) + }) + + it('keeps retrying GitHub after the mirror lacks the asset', async () => { + // Mirror 404, GitHub 504, GitHub ok: the mirror's miss must not cost + // GitHub its retries, and the 504 must still be waited out. + mockDownloadTool + .mockRejectedValueOnce(httpError(404)) + .mockRejectedValueOnce(httpError(504)) + .mockResolvedValueOnce('/tmp/sfw') + + await expect(downloadToolWithRetry([MIRROR, GITHUB])).resolves.toBe( + '/tmp/sfw', + ) + expect(mockDownloadTool.mock.calls).toEqual([[MIRROR], [GITHUB], [GITHUB]]) + expect(sleepDelays).toEqual([30_000]) + }) + + it('drops an origin that is forbidden and reports the outage, not the 403', async () => { + // GitHub 504 then mirror 403: GitHub keeps its full budget on its own, + // and the error thrown is the 504 that explains why the job failed. + mockDownloadTool.mockImplementation(async (url: string) => { + throw httpError(url === MIRROR ? 403 : 504) + }) + + await expect(downloadToolWithRetry([GITHUB, MIRROR])).rejects.toThrow( + 'Unexpected HTTP response: 504', + ) + expect(mockDownloadTool.mock.calls).toEqual([ + [GITHUB], + [MIRROR], + [GITHUB], + [GITHUB], + ]) + expect(sleepDelays).toEqual([30_000, 60_000]) + }) + + it('tries the other origin immediately when a retry cannot help', async () => { + // A 404 from the mirror says nothing about GitHub: no backoff, one + // immediate try of the other origin. + mockDownloadTool + .mockRejectedValueOnce(httpError(404)) + .mockResolvedValueOnce('/tmp/sfw') + + await expect(downloadToolWithRetry([MIRROR, GITHUB])).resolves.toBe( + '/tmp/sfw', + ) + expect(mockDownloadTool.mock.calls).toEqual([[MIRROR], [GITHUB]]) + expect(sleepDelays).toEqual([]) + }) + + it('gives up once every origin reported a missing asset', async () => { + mockDownloadTool.mockRejectedValue(httpError(404)) + + await expect(downloadToolWithRetry([MIRROR, GITHUB])).rejects.toThrow( + 'Unexpected HTTP response: 404', + ) + expect(mockDownloadTool).toHaveBeenCalledTimes(2) + expect(sleepDelays).toEqual([]) + }) +}) + +describe('firewallDownloadUrls', () => { + it('gives the free edition both origins, GitHub first', () => { + expect( + firewallDownloadUrls( + 'free', + 'sfw-free', + 'v1.15.2', + 'sfw-free-linux-x86_64', + ), + ).toEqual([ + 'https://github.com/SocketDev/sfw-free/releases/download/v1.15.2/sfw-free-linux-x86_64', + 'https://install.socket.dev/firewall/dl/v1.15.2/sfw-free-linux-x86_64', + ]) + }) + + it('keeps the enterprise edition GitHub-only', () => { + expect( + firewallDownloadUrls( + 'enterprise', + 'firewall-release', + 'v1.15.2', + 'sfw-windows-x86_64.exe', + ), + ).toEqual([ + 'https://github.com/SocketDev/firewall-release/releases/download/v1.15.2/sfw-windows-x86_64.exe', + ]) + }) +}) + +describe('shuffledIndexes', () => { + it('permutes two indexes by the coin flip', () => { + expect(shuffledIndexes(2, () => 0.9)).toEqual([0, 1]) + expect(shuffledIndexes(2, () => 0.1)).toEqual([1, 0]) + }) + + it('permutes three indexes from the injected sequence', () => { + const draws = [0.9, 0.1] + expect(shuffledIndexes(3, () => draws.shift() ?? 0)).toEqual([1, 0, 2]) + }) + + it('returns every index exactly once', () => { + for (let n = 0; n <= 5; n++) { + expect(shuffledIndexes(n).toSorted((a, b) => a - b)).toEqual( + Array.from({ length: n }, (_, i) => i), + ) + } + }) + + it('does not consult the random source for a single index', () => { + const random = vi.fn(() => 0.1) + expect(shuffledIndexes(1, random)).toEqual([0]) + expect(random).not.toHaveBeenCalled() + }) })