diff --git a/docker-compose.yml b/docker-compose.yml index 273963cef0..728373d6d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -703,6 +703,11 @@ services: - gittensory-backups:/backups - ./scripts/backup.sh:/backup.sh:ro - ./scripts/verify-backup.sh:/verify-backup.sh:ro + # Shared url_decode/pgpass_escape helpers (#2910), sourced by both scripts above via + # `. "$(dirname "$0")/selfhost-pg-url.sh"` -- must be mounted at container root alongside them so that + # dirname-relative resolution finds it at /selfhost-pg-url.sh, the same way it finds + # scripts/selfhost-pg-url.sh when either script is run directly from a repo checkout. + - ./scripts/selfhost-pg-url.sh:/selfhost-pg-url.sh:ro # `docker compose run --rm backup sh /backup.sh` (or /verify-backup.sh) REPLACES `command:`, not # `entrypoint:`, so the package install must live in the entrypoint or an on-demand run gets a bare # container with no pg_restore/sqlite3/psql. The entrypoint installs packages once, then `exec "$@"` diff --git a/scripts/backup.sh b/scripts/backup.sh index c08ba8cae8..976c0c28e6 100644 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -5,6 +5,9 @@ # Backups land in the `gittensory-backups` volume at /backups/{postgres,sqlite,qdrant}. set -eu +# shellcheck source=selfhost-pg-url.sh +. "$(dirname "$0")/selfhost-pg-url.sh" + normalize_backup_retain() { retain_value=$1 case "$retain_value" in @@ -54,32 +57,6 @@ mkdir -p "$OUT/postgres" "$OUT/sqlite" "$OUT/qdrant" # (never delete the last good backup) and still exit non-zero at the end (fail loudly). SQLITE_BACKUP_FAILED=0 -# Percent-decodes a URI userinfo component (RFC 3986). Deliberately does NOT treat '+' as a space -- that -# convention is specific to application/x-www-form-urlencoded query values, not URI userinfo, where '+' is -# an ordinary sub-delims character allowed unencoded; the only caller of this function decodes a password -# extracted from the userinfo section, and a literal '+' there must stay a '+', not become a space. -url_decode() { - printf '%s' "$1" | awk ' - BEGIN { for (i = 0; i < 256; i++) hex[sprintf("%02X", i)] = sprintf("%c", i); } - { - out = ""; - for (i = 1; i <= length($0); i++) { - c = substr($0, i, 1); - if (c == "%" && i + 2 <= length($0)) { - h = toupper(substr($0, i + 1, 2)); - if (h in hex) { out = out hex[h]; i += 2; } else { out = out c; } - } else { - out = out c; - } - } - printf "%s", out; - }' -} - -pgpass_escape() { - printf '%s' "$1" | sed 's/\\/\\\\/g; s/:/\\:/g' -} - json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' } @@ -136,6 +113,13 @@ write_manifest() { # this script makes, and is deleted immediately after via the `cleanup` trap, so there's no scoped value # in re-deriving the exact host/port/dbname libpq will resolve -- which the query string can override # anyway -- just to match them precisely). +# +# NOT extracted alongside url_decode/pgpass_escape into selfhost-pg-url.sh (#2910): verify-backup.sh's +# pg_connect_arg() shares this exact URI-parsing algorithm, but the two are not safe to collapse into one +# function -- this one reads $PG_DB from a global and runs once per invocation; pg_connect_arg() takes the +# URL as an argument, unsets PGPASSFILE at the top of every call (a reentrancy guard this script doesn't +# need, since it only ever connects once), and tracks a LIST of created passfiles instead of one. See +# verify-backup.sh's pg_connect_arg for the full rationale. prepare_pg_env() { pg_rest=${PG_DB#postgres://} pg_rest=${pg_rest#postgresql://} diff --git a/scripts/deploy-selfhost-image.sh b/scripts/deploy-selfhost-image.sh index 8452155ea8..d9e5cc5ca2 100755 --- a/scripts/deploy-selfhost-image.sh +++ b/scripts/deploy-selfhost-image.sh @@ -17,97 +17,9 @@ SERVICE="${SELFHOST_SERVICE:-gittensory}" HEALTH_TIMEOUT_SECONDS="${SELFHOST_HEALTH_TIMEOUT_SECONDS:-180}" DEFAULT_IMAGE="ghcr.io/jsonbored/gittensory-selfhost:latest" -require_cmd() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "error: required command not found: $1" >&2 - exit 1 - fi -} - -env_get() { - local key="$1" - local file="${2:-$ENV_FILE}" - - [ -f "$file" ] || return 1 - - awk -v key="$key" ' - /^[[:space:]]*(#|$)/ { next } - { - line = $0 - sub(/^[[:space:]]*/, "", line) - if (line !~ "^" key "[[:space:]]*=") { - next - } - sub(/^[^=]*=/, "", line) - sub(/^[[:space:]]*/, "", line) - sub(/[[:space:]]*$/, "", line) - if (length(line) >= 2) { - first = substr(line, 1, 1) - last = substr(line, length(line), 1) - if ((first == "\"" && last == "\"") || (first == "'\''" && last == "'\''")) { - line = substr(line, 2, length(line) - 2) - } - } - print line - found = 1 - exit - } - END { exit found ? 0 : 1 } - ' "$file" -} - -env_put() { - local key="$1" - local value="$2" - local file="${3:-$ENV_FILE}" - local dir base tmp - - touch "$file" - dir="$(dirname "$file")" - base="$(basename "$file")" - tmp="$(mktemp "$dir/.${base}.tmp.XXXXXX")" - awk -v key="$key" -v value="$value" ' - BEGIN { written = 0 } - { - line = $0 - sub(/^[[:space:]]*/, "", line) - if (line ~ "^" key "[[:space:]]*=") { - print key "=" value - written = 1 - } else { - print $0 - } - } - END { - if (!written) { - print key "=" value - } - } - ' "$file" >"$tmp" - cat "$tmp" >"$file" - rm -f "$tmp" -} - -compose_file_args() { - local files=() - local file - - if [ -n "${SELFHOST_COMPOSE_FILES:-}" ]; then - # shellcheck disable=SC2206 - files=(${SELFHOST_COMPOSE_FILES}) - else - files=(docker-compose.yml) - [ -f docker-compose.override.yml ] && files+=(docker-compose.override.yml) - fi - - for file in "${files[@]}"; do - if [ ! -f "$file" ]; then - echo "error: compose file not found: $file" >&2 - exit 1 - fi - printf '%s\n' -f "$file" - done -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/selfhost-deploy-common.sh +. "$SCRIPT_DIR/lib/selfhost-deploy-common.sh" resolve_image() { local env_file_image diff --git a/scripts/deploy-selfhost-prebuilt.sh b/scripts/deploy-selfhost-prebuilt.sh index fb55087e91..88984ca5a9 100755 --- a/scripts/deploy-selfhost-prebuilt.sh +++ b/scripts/deploy-selfhost-prebuilt.sh @@ -16,95 +16,9 @@ SERVICE="${SELFHOST_SERVICE:-gittensory}" SKIP_SENTRY_UPLOAD="${SELFHOST_SKIP_SENTRY_UPLOAD:-0}" SENTRY_CLI_PACKAGE="${SENTRY_CLI_PACKAGE:-@sentry/cli@3.6.0}" -require_cmd() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "error: required command not found: $1" >&2 - exit 1 - fi -} - -env_get() { - local key="$1" - local file="${2:-$ENV_FILE}" - - [ -f "$file" ] || return 1 - - awk -v key="$key" ' - /^[[:space:]]*(#|$)/ { next } - { - line = $0 - sub(/^[[:space:]]*/, "", line) - if (line !~ "^" key "[[:space:]]*=") { - next - } - sub(/^[^=]*=/, "", line) - sub(/^[[:space:]]*/, "", line) - sub(/[[:space:]]*$/, "", line) - if (length(line) >= 2) { - first = substr(line, 1, 1) - last = substr(line, length(line), 1) - if ((first == "\"" && last == "\"") || (first == "'\''" && last == "'\''")) { - line = substr(line, 2, length(line) - 2) - } - } - print line - found = 1 - exit - } - END { exit found ? 0 : 1 } - ' "$file" -} - -env_put() { - local key="$1" - local value="$2" - local file="${3:-$ENV_FILE}" - local tmp - - touch "$file" - tmp="$(mktemp)" - awk -v key="$key" -v value="$value" ' - BEGIN { written = 0 } - { - line = $0 - sub(/^[[:space:]]*/, "", line) - if (line ~ "^" key "[[:space:]]*=") { - print key "=" value - written = 1 - } else { - print $0 - } - } - END { - if (!written) { - print key "=" value - } - } - ' "$file" >"$tmp" - cat "$tmp" >"$file" - rm -f "$tmp" -} - -compose_file_args() { - local files=() - local file - - if [ -n "${SELFHOST_COMPOSE_FILES:-}" ]; then - # shellcheck disable=SC2206 - files=(${SELFHOST_COMPOSE_FILES}) - else - files=(docker-compose.yml) - [ -f docker-compose.override.yml ] && files+=(docker-compose.override.yml) - fi - - for file in "${files[@]}"; do - if [ ! -f "$file" ]; then - echo "error: compose file not found: $file" >&2 - exit 1 - fi - printf '%s\n' -f "$file" - done -} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/selfhost-deploy-common.sh +. "$SCRIPT_DIR/lib/selfhost-deploy-common.sh" run_node_build() { local uid gid diff --git a/scripts/lib/selfhost-deploy-common.sh b/scripts/lib/selfhost-deploy-common.sh new file mode 100644 index 0000000000..3a9f25ec38 --- /dev/null +++ b/scripts/lib/selfhost-deploy-common.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Shared helpers for the self-host deploy scripts (deploy-selfhost-image.sh, deploy-selfhost-prebuilt.sh). +# Sourced, not executed: this file has no shebang-driven side effects and defines functions only. +# Both callers set ENV_FILE before sourcing this; env_get/env_put fall back to it when no file arg is given. + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +env_get() { + local key="$1" + local file="${2:-$ENV_FILE}" + + [ -f "$file" ] || return 1 + + awk -v key="$key" ' + /^[[:space:]]*(#|$)/ { next } + { + line = $0 + sub(/^[[:space:]]*/, "", line) + if (line !~ "^" key "[[:space:]]*=") { + next + } + sub(/^[^=]*=/, "", line) + sub(/^[[:space:]]*/, "", line) + sub(/[[:space:]]*$/, "", line) + if (length(line) >= 2) { + first = substr(line, 1, 1) + last = substr(line, length(line), 1) + if ((first == "\"" && last == "\"") || (first == "'\''" && last == "'\''")) { + line = substr(line, 2, length(line) - 2) + } + } + print line + found = 1 + exit + } + END { exit found ? 0 : 1 } + ' "$file" +} + +# Same-directory temp file (not the system tmpdir): guarantees `cat "$tmp" >"$file"` never crosses a +# filesystem boundary, which a plain `mktemp` could when $ENV_FILE lives on a different mount than the +# default tmp directory (#2910 -- this was previously only true for deploy-selfhost-image.sh's copy of +# this function; deploy-selfhost-prebuilt.sh's copy used a plain `mktemp` with no documented reason for +# the difference, so consolidating adopts the more defensive behavior for both callers). +env_put() { + local key="$1" + local value="$2" + local file="${3:-$ENV_FILE}" + local dir base tmp + + touch "$file" + dir="$(dirname "$file")" + base="$(basename "$file")" + tmp="$(mktemp "$dir/.${base}.tmp.XXXXXX")" + awk -v key="$key" -v value="$value" ' + BEGIN { written = 0 } + { + line = $0 + sub(/^[[:space:]]*/, "", line) + if (line ~ "^" key "[[:space:]]*=") { + print key "=" value + written = 1 + } else { + print $0 + } + } + END { + if (!written) { + print key "=" value + } + } + ' "$file" >"$tmp" + cat "$tmp" >"$file" + rm -f "$tmp" +} + +compose_file_args() { + local files=() + local file + + if [ -n "${SELFHOST_COMPOSE_FILES:-}" ]; then + # shellcheck disable=SC2206 + files=(${SELFHOST_COMPOSE_FILES}) + else + files=(docker-compose.yml) + [ -f docker-compose.override.yml ] && files+=(docker-compose.override.yml) + fi + + for file in "${files[@]}"; do + if [ ! -f "$file" ]; then + echo "error: compose file not found: $file" >&2 + exit 1 + fi + printf '%s\n' -f "$file" + done +} diff --git a/scripts/selfhost-pg-url.sh b/scripts/selfhost-pg-url.sh new file mode 100644 index 0000000000..15034a21cf --- /dev/null +++ b/scripts/selfhost-pg-url.sh @@ -0,0 +1,44 @@ +# shellcheck shell=sh +# Shared Postgres-URL credential helpers for backup.sh and verify-backup.sh (#2910). Sourced, not executed: +# no shebang side effects, functions only. Lives as a SIBLING file (not a lib/ subdirectory) because both +# callers are bind-mounted individually at container root (/backup.sh, /verify-backup.sh) with no shared +# /lib directory — `. "$(dirname "$0")/selfhost-pg-url.sh"` resolves correctly either way: to this file's +# path in the repo checkout when run directly (tests), or to /selfhost-pg-url.sh in the backup container +# (see docker-compose.yml's matching bind-mount for this file). +# +# Deliberately does NOT include prepare_pg_env()/pg_connect_arg() — despite sharing this same URI-parsing +# algorithm, backup.sh's and verify-backup.sh's versions differ in their PGPASSFILE lifecycle in ways that +# are not safe to collapse into one function: backup.sh's prepare_pg_env() reads the URL from a single +# global ($PG_DB) and runs once per script invocation, while verify-backup.sh's pg_connect_arg() takes the +# URL as an argument, `unset PGPASSFILE` at the top of every call (a reentrancy guard against a stale +# PGPASSFILE from a PREVIOUS call for a different URL leaking into this one), and tracks every passfile it +# creates in a list ($PG_PASSFILES) for its cleanup trap to sweep, since it may run several times per +# invocation (scratch/live/scratch/scratch in the scratch-restore flow) — backup.sh only ever creates one. +# Forcing these into a single shared function risks losing that reentrancy guard, which would leak a +# password across connections in verify-backup.sh's multi-URL flow. + +# Percent-decodes a URI userinfo component (RFC 3986). Deliberately does NOT treat '+' as a space -- that +# convention is specific to application/x-www-form-urlencoded query values, not URI userinfo, where '+' is +# an ordinary sub-delims character allowed unencoded; the only callers decode a password extracted from the +# userinfo section or a query-string value, and a literal '+' there must stay a '+', not become a space. +url_decode() { + printf '%s' "$1" | awk ' + BEGIN { for (i = 0; i < 256; i++) hex[sprintf("%02X", i)] = sprintf("%c", i); } + { + out = ""; + for (i = 1; i <= length($0); i++) { + c = substr($0, i, 1); + if (c == "%" && i + 2 <= length($0)) { + h = toupper(substr($0, i + 1, 2)); + if (h in hex) { out = out hex[h]; i += 2; } else { out = out c; } + } else { + out = out c; + } + } + printf "%s", out; + }' +} + +pgpass_escape() { + printf '%s' "$1" | sed 's/\\/\\\\/g; s/:/\\:/g' +} diff --git a/scripts/verify-backup.sh b/scripts/verify-backup.sh index 14110cf58e..1b91833a6e 100644 --- a/scripts/verify-backup.sh +++ b/scripts/verify-backup.sh @@ -20,31 +20,8 @@ cleanup() { } trap cleanup EXIT HUP INT TERM -# Percent-decodes a URI userinfo component (RFC 3986). Deliberately does NOT treat '+' as a space -- that -# convention is specific to application/x-www-form-urlencoded query values, not URI userinfo, where '+' is -# an ordinary sub-delims character allowed unencoded; the only caller of this function decodes a password -# extracted from the userinfo section, and a literal '+' there must stay a '+', not become a space. -url_decode() { - printf '%s' "$1" | awk ' - BEGIN { for (i = 0; i < 256; i++) hex[sprintf("%02X", i)] = sprintf("%c", i); } - { - out = ""; - for (i = 1; i <= length($0); i++) { - c = substr($0, i, 1); - if (c == "%" && i + 2 <= length($0)) { - h = toupper(substr($0, i + 1, 2)); - if (h in hex) { out = out hex[h]; i += 2; } else { out = out c; } - } else { - out = out c; - } - } - printf "%s", out; - }' -} - -pgpass_escape() { - printf '%s' "$1" | sed 's/\\/\\\\/g; s/:/\\:/g' -} +# shellcheck source=selfhost-pg-url.sh +. "$(dirname "$0")/selfhost-pg-url.sh" # Strips the password from a postgres(ql):// URI -- from EITHER the userinfo (user:password@host) or a # `password=` libpq query-string parameter (postgresql://user@host/db?password=secret is equally valid @@ -58,6 +35,11 @@ pgpass_escape() { # unsets PGPASSFILE first, so a previous call's password can never leak into a connection for a URL that # doesn't have one of its own. Sets $PG_SANITIZED_URL; exports PGPASSFILE (tracked in $PG_PASSFILES for # cleanup) if the given URL had a password. +# +# NOT extracted into selfhost-pg-url.sh alongside url_decode/pgpass_escape (#2910): despite sharing the +# same URI-parsing algorithm as backup.sh's prepare_pg_env(), the two are not safe to collapse into one +# function given the PGPASSFILE-lifecycle differences described above (arg vs. global URL, unset-at-start +# reentrancy guard, a tracked LIST of passfiles vs. one). See prepare_pg_env in backup.sh. pg_connect_arg() { # Cleared up front, not just when this URL turns out to have no password: any helper command invoked # below (e.g. url_decode) would otherwise inherit a still-exported PGPASSFILE left over from a PREVIOUS