From d72dc732630bd1ec14ed04a2c838872909f5bbc1 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Fri, 29 May 2026 17:59:54 +0100 Subject: [PATCH 1/2] fix: dedup pageindex citations->sections in retriever The page endpoint can emit several citations whose distinct page ranges all anchor to the SAME first section_id (overlapping sections, or the literal same range cited N times). _sections_from_citations mapped each to its own RetrievedSection, so the FinanceBench "sec_363... x5" miss became five identical sections and deflated precision@5. Dedup by section_id, keeping the first occurrence per id in order. Citations with no section_id keep their own row (their page anchor is still distinct). The k-cap is applied AFTER dedup so a spray of duplicates can't crowd out genuinely distinct sections. Tests: five duplicate sec_363 citations + one distinct -> two sections (first occurrence kept), dedup-before-cap with k=2, and empty-section-id rows not collapsed. Existing mapping/usage/k-cap/error tests stay green. --- .../retrievers/vectorless_pageindex.py | 25 ++++++- tests/test_vectorless_pageindex.py | 69 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/vectorless_bench/retrievers/vectorless_pageindex.py b/src/vectorless_bench/retrievers/vectorless_pageindex.py index 9430c8e..a432afb 100644 --- a/src/vectorless_bench/retrievers/vectorless_pageindex.py +++ b/src/vectorless_bench/retrievers/vectorless_pageindex.py @@ -180,14 +180,37 @@ def _sections_from_citations( """Project the page endpoint's citation list onto the bench's section shape. Order is preserved (engine sorts by start_page) and capped at k so head-to-head fairness against k=5 baselines is intact. + + Deduplication by section_id is load-bearing for precision. A page-based + answer can emit several citations whose distinct page ranges all anchor + to the SAME first section_id (overlapping sections, or — before the + engine-side fix — the literal same range cited N times). Mapping each to + its own RetrievedSection produced N identical sections and tanked + precision@k (the FinanceBench "sec_363... ×5" miss). We keep only the + first section per distinct section_id, preserving order. + + Citations with no section_id still carry a distinct page anchor, so they + are NOT collapsed together — only repeated, non-empty section ids are + deduped. The k-cap is applied AFTER dedup so a spray of duplicates can't + crowd out genuinely distinct sections. """ out: List[RetrievedSection] = [] - for c in citations[:k]: + seen_ids: set[str] = set() + for c in citations: + if len(out) >= k: + break sec_ids = c.get("section_ids") or [] # First section_id anchors the citation in the structural tree # for path matching; the page range stays in `page` so the # bench's per-page anchors still hit. sid = str(sec_ids[0]) if sec_ids else "" + # Dedup on non-empty ids only: a repeated section id is the + # precision-killer we are removing. Empty-id citations keep their + # own row (their page anchor is still distinct). + if sid: + if sid in seen_ids: + continue + seen_ids.add(sid) start_page = c.get("start_page") try: start_page_int = int(start_page) if start_page is not None else None diff --git a/tests/test_vectorless_pageindex.py b/tests/test_vectorless_pageindex.py index d090a53..01784b7 100644 --- a/tests/test_vectorless_pageindex.py +++ b/tests/test_vectorless_pageindex.py @@ -166,6 +166,75 @@ def test_k_caps_returned_sections(monkeypatch): assert len(res.sections) == 1 # two citations available, k=1 caps to one +def _payload_with_duplicate_section() -> dict: + """A response shaped like the FinanceBench 'sec_363... ×5' miss: several + citations whose distinct page ranges all anchor to the SAME first + section_id. Without dedup these became 5 identical RetrievedSections and + deflated precision@5.""" + p = _payload() + p["citations"] = [ + {"start_page": 100, "end_page": 101, "section_ids": ["sec_363"], "quote": "a"}, + {"start_page": 102, "end_page": 103, "section_ids": ["sec_363"], "quote": "b"}, + {"start_page": 104, "end_page": 105, "section_ids": ["sec_363"], "quote": "c"}, + {"start_page": 106, "end_page": 107, "section_ids": ["sec_363"], "quote": "d"}, + {"start_page": 108, "end_page": 109, "section_ids": ["sec_363"], "quote": "e"}, + {"start_page": 5, "end_page": 6, "section_ids": ["sec_50"], "quote": "f"}, + ] + return p + + +def test_duplicate_section_ids_are_deduped(monkeypatch): + r = _build(monkeypatch) + r._http = _FakeHTTP(_FakeResponse(200, _payload_with_duplicate_section())) + r._doc_ids["AMZN"] = "doc_x" + r._paths["AMZN"] = {} + + q = Question(qid="q1", doc_id="AMZN", question="q?") + res = r.retrieve(q, k=5, cold=True) + + # sec_363 cited five times collapses to ONE section; sec_50 is the other. + ids = [s.section_id for s in res.sections] + assert ids == ["sec_363", "sec_50"], ids + # No section id repeats. + assert len(ids) == len(set(ids)) + # The kept sec_363 row is the FIRST occurrence (page 100), not a later one. + assert res.sections[0].page == 100 + assert res.sections[0].content == "a" + + +def test_dedup_runs_before_k_cap(monkeypatch): + # Five duplicate sec_363 citations + one distinct sec_50. With k=2 the + # duplicates must not crowd out sec_50: dedup first, THEN cap. + r = _build(monkeypatch) + r._http = _FakeHTTP(_FakeResponse(200, _payload_with_duplicate_section())) + r._doc_ids["AMZN"] = "doc_x" + r._paths["AMZN"] = {} + + q = Question(qid="q1", doc_id="AMZN", question="q?") + res = r.retrieve(q, k=2, cold=True) + ids = [s.section_id for s in res.sections] + assert ids == ["sec_363", "sec_50"], ids + + +def test_empty_section_ids_not_collapsed(monkeypatch): + # Citations with no section_id keep their own row — they still carry a + # distinct page anchor — so they are NOT deduped together. + p = _payload() + p["citations"] = [ + {"start_page": 10, "end_page": 11, "section_ids": [], "quote": "x"}, + {"start_page": 20, "end_page": 21, "section_ids": [], "quote": "y"}, + ] + r = _build(monkeypatch) + r._http = _FakeHTTP(_FakeResponse(200, p)) + r._doc_ids["AMZN"] = "doc_x" + r._paths["AMZN"] = {} + + q = Question(qid="q1", doc_id="AMZN", question="q?") + res = r.retrieve(q, k=5, cold=True) + assert len(res.sections) == 2 + assert [s.page for s in res.sections] == [10, 20] + + def test_http_error_is_recorded_not_raised(monkeypatch): r = _build(monkeypatch) r._http = _FakeHTTP(_FakeResponse(404, {})) From bc430baca509c9de18168333fb22660a6c4a55b6 Mon Sep 17 00:00:00 2001 From: Halleluyah Oludele Date: Tue, 2 Jun 2026 10:28:03 +0100 Subject: [PATCH 2/2] bench: GLM/no-OpenAI three-way + autonomous GCE run + resilient image build - vector_rag: local sentence-transformers embedding path (st:) so the baseline runs with no OpenAI key; pyproject adds sentence-transformers to the vector extra. - Dockerfile: PageIndex requirements install is now best-effort (upstream pins hit ResolutionImpossible) so it can't sink the whole image; the pageindex baseline is optional. - configs/financebench_threeway.yaml: vectorless_pageindex vs vector_rag (local embeddings) vs bm25, glm-4.6, limit 150. - deploy: load_secrets also extracts an OpenAI key if present; startup-autonomous.sh runs the bench from VM metadata (no SSH) and uploads results+log to GCS; run_on_gce_autonomous.ps1 local launcher. --- Dockerfile | 12 +- configs/financebench_threeway.yaml | 48 +++++++ deploy/gcp/run_on_gce_autonomous.ps1 | 122 ++++++++++++++++++ deploy/gcp/startup-autonomous.sh | 60 +++++++++ deploy/load_secrets.py | 101 +++++++++++++++ pyproject.toml | 6 +- src/vectorless_bench/retrievers/vector_rag.py | 34 ++++- 7 files changed, 374 insertions(+), 9 deletions(-) create mode 100644 configs/financebench_threeway.yaml create mode 100644 deploy/gcp/run_on_gce_autonomous.ps1 create mode 100644 deploy/gcp/startup-autonomous.sh create mode 100644 deploy/load_secrets.py diff --git a/Dockerfile b/Dockerfile index 6464cbe..a22019d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,9 +23,15 @@ RUN pip install ".[llm,vector,bm25,data,viz]" \ || echo "WARN: vectorless-sdk not installed — mount it for the vectorless system") # Vendor PageIndex's actual repo (not on PyPI) and install its requirements, so -# the pageindex baseline runs their real tree builder. -RUN git clone --depth 1 https://github.com/VectifyAI/PageIndex.git /opt/PageIndex \ - && pip install -r /opt/PageIndex/requirements.txt +# the pageindex baseline runs their real tree builder. Best-effort: upstream's +# pinned requirements.txt sometimes has internal conflicts (e.g. litellm vs +# python-dotenv → ResolutionImpossible). The clone is cheap and only the +# `pageindex` baseline needs these deps, so a failed install must NOT break the +# image for every other system. Retry without the broken pins, else skip. +RUN git clone --depth 1 https://github.com/VectifyAI/PageIndex.git /opt/PageIndex || true; \ + pip install -r /opt/PageIndex/requirements.txt \ + || pip install litellm pymupdf PyPDF2 python-dotenv pyyaml tiktoken openai \ + || echo "WARN: PageIndex deps skipped — the 'pageindex' baseline will be unavailable" # results land here; mount a volume so they survive the container VOLUME ["/results"] diff --git a/configs/financebench_threeway.yaml b/configs/financebench_threeway.yaml new file mode 100644 index 0000000..741bb6a --- /dev/null +++ b/configs/financebench_threeway.yaml @@ -0,0 +1,48 @@ +# Full FinanceBench three-way: the Vectorless engine (PageIndex mode, the +# path improved by the citation reframe + GLM-4.6 pricing) vs a real vector +# RAG baseline vs the BM25 lexical floor. +# +# Prereqs (see deploy/README.md): +# 1. The Cloud Run engine redeployed from the fixed code (llmgate v0.3.0 so +# $/query != 0; reframed pageindex prompt), caches OFF for cold cost. +# 2. .env with VECTORLESS_BASE_URL / VECTORLESS_API_KEY (load_secrets.py), +# OPENAI_API_KEY (vector_rag embeddings + gen), and VLBENCH_PG_DSN +# (the bundled pgvector Postgres provides this on the VM). +# 3. FinanceBench docs fetched (run_on_gce.sh does this on the VM). +dataset: financebench +dataset_params: + docs_dir: data/financebench/docs + +systems: + - vectorless_pageindex + - vector_rag + - bm25 + +k: 5 +repeats: 1 # quality/cost/latency from one cold pass; bump for determinism +cold: true +limit: 150 # full open subset +sample_seed: 0 +out_dir: runs + +# Query-time model recorded in the manifest. The engine answers on glm-4.6 +# (the deployed service is configured for z.ai's Anthropic-compatible GLM +# endpoint). No OpenAI anywhere: vector_rag embeds with a LOCAL +# sentence-transformers model (free, deterministic), judging is off. +model: glm-4.6 +embedding_model: st:BAAI/bge-small-en-v1.5 +judge: false + +system_params: + vectorless_pageindex: + org: vlbench + ingest_timeout: 5400 + query_timeout: 240 + max_hops: 6 + page_content_limit: 16000 + server_cache_disabled: true # you attest the deployed server runs cold + vector_rag: + backend: memory # in-RAM cosine; no Postgres, no API key + embedding_model: st:BAAI/bge-small-en-v1.5 + chunk_tokens: 512 + overlap_tokens: 64 diff --git a/deploy/gcp/run_on_gce_autonomous.ps1 b/deploy/gcp/run_on_gce_autonomous.ps1 new file mode 100644 index 0000000..f636375 --- /dev/null +++ b/deploy/gcp/run_on_gce_autonomous.ps1 @@ -0,0 +1,122 @@ +<# + Autonomous (fire-and-forget) GCE runner for the full FinanceBench run. + + Unlike run_on_gce.ps1 (which runs the benchmark attached over SSH and pulls + results back to the local machine), this: + - provisions the VM with cloud-platform scope and a long max-run-duration + (so a ~14h GLM ingest of 150 10-Ks can finish), termination-action=DELETE + as the backstop, + - ships the bundle + .env (secrets from Secret Manager), + - launches the build+download+run DETACHED on the VM (setsid/nohup), so this + script returns immediately and the run survives the local session ending, + - the remote job uploads results to gs://BUCKET/NAME/ on completion and + best-effort self-deletes the VM. + + Watch progress later with: + gcloud compute ssh NAME --tunnel-through-iap --command "tail -f ~/vlbench-run.log" + Results land in: gs://BUCKET/NAME/ (fetch_results.ps1 / .sh) +#> +param( + [string]$Project = "project-03250746-ec5b-4198-990", + [string]$Zone = "us-central1-a", + [string]$Machine = "e2-standard-4", + [string]$Name = "vlbench-full-$(Get-Date -Format yyyyMMdd-HHmmss)", + [string]$Config = "configs/financebench_threeway.yaml", + [string]$Secret = "server-config", + [string]$BaseUrl = "https://vectorless-server-2rzh3kctga-uc.a.run.app", + [string]$Bucket = "gs://vectorless-engine-us-central1", + [string]$Tags = "dokploy", + [int]$DiskGb = 100, + [string]$MaxRun = "20h" +) +$ErrorActionPreference = "Stop" +$repo = (Resolve-Path "$PSScriptRoot\..\..").Path +Set-Location $repo + +function GcloudSsh([string]$cmd) { + "y" | gcloud compute ssh $Name --project=$Project --zone=$Zone --tunnel-through-iap --command="$cmd" +} + +Write-Host ">> vendoring the vectorless SDK into the build context" +$sdkSrc = (Resolve-Path "$repo\..\vectorless-sdk\python").Path +$dest = "$repo\vendor\vectorless-sdk" +if (Test-Path $dest) { Remove-Item -Recurse -Force $dest } +New-Item -ItemType Directory -Force (Split-Path $dest) | Out-Null +Copy-Item -Recurse $sdkSrc $dest +Remove-Item -Recurse -Force "$dest\.venv","$dest\dist","$dest\build" -ErrorAction SilentlyContinue + +Write-Host ">> reading secrets from Secret Manager ($Secret) into .env" +python deploy\load_secrets.py --project $Project --secret $Secret --base-url $BaseUrl --out .env +if ($LASTEXITCODE -ne 0) { throw "load_secrets failed" } + +# Note: the project's default compute service account already holds +# roles/storage.objectAdmin on the results bucket, so the VM (cloud-platform +# scope) can upload results — no IAM change needed here. + +Write-Host ">> creating VM $Name ($Machine, $Zone, disk ${DiskGb}GB, max-run $MaxRun)" +gcloud compute instances create $Name --project=$Project --zone=$Zone --machine-type=$Machine ` + --image-family=ubuntu-2204-lts --image-project=ubuntu-os-cloud --boot-disk-size="${DiskGb}GB" ` + --scopes=cloud-platform ` + --tags=$Tags --max-run-duration=$MaxRun --instance-termination-action=DELETE ` + --metadata-from-file=startup-script="deploy\gcp\startup-script.sh" +if ($LASTEXITCODE -ne 0) { throw "VM create failed" } + +try { + Write-Host ">> waiting for Docker on the VM (caching host key)" + $ready = $false + for ($i = 0; $i -lt 40; $i++) { + GcloudSsh "sudo docker ps >/dev/null 2>&1 && test -f /var/run/vlbench-ready" 2>$null + if ($LASTEXITCODE -eq 0) { $ready = $true; Write-Host " ready"; break } + Start-Sleep -Seconds 15 + } + if (-not $ready) { throw "Docker never came up on the VM" } + + Write-Host ">> shipping bundle + .env over IAP" + $tgzName = "vlbench-$Name.tgz" + tar --exclude=.git --exclude=runs --exclude="data/financebench/docs" ` + --exclude=.venv --exclude=__pycache__ --exclude="*.pyc" --exclude="*.tgz" ` + -czf $tgzName -C $repo . + gcloud compute scp $tgzName "${Name}:vlbench.tgz" --project=$Project --zone=$Zone --tunnel-through-iap + if ($LASTEXITCODE -ne 0) { throw "scp bundle failed" } + gcloud compute scp ".env" "${Name}:vlbench.env" --project=$Project --zone=$Zone --tunnel-through-iap + if ($LASTEXITCODE -ne 0) { throw "scp .env failed" } + + # Remote runner: build -> download docs -> run -> upload to GCS -> self-delete. + # Single-quoted here-string so PowerShell does NOT evaluate bash $(...) — the + # PS variables are injected via __PLACEHOLDER__ replacement afterwards. + $runnerTemplate = @' +#!/usr/bin/env bash +set -uxo pipefail +exec > ~/vlbench-run.log 2>&1 +# Always shut the VM down when this script ends (success OR failure) so a broken +# build can't leave it running until the max-run-duration cap. +trap 'rc=$?; echo "EXIT rc=$rc $(date -u)"; sudo gcloud compute instances delete __NAME__ --zone=__ZONE__ --quiet || sudo poweroff' EXIT +echo "START $(date -u)" +set -e # fail LOUD: abort on first error rather than producing an empty bucket +rm -rf ~/vlbench && mkdir -p ~/vlbench && tar -xzf ~/vlbench.tgz -C ~/vlbench +cd ~/vlbench && cp ~/vlbench.env .env +sudo docker compose build +sudo docker compose run --rm --entrypoint python bench scripts/download_financebench.py +sudo docker compose run --rm bench run --config __CONFIG__ --out /results +echo "RUN DONE $(date -u); uploading to __BUCKET__/__NAME__/" +sudo gcloud storage cp -r results/* "__BUCKET__/__NAME__/" +echo "UPLOADED $(date -u)" +'@ + $runner = $runnerTemplate.Replace('__CONFIG__', $Config).Replace('__BUCKET__', $Bucket).Replace('__NAME__', $Name).Replace('__ZONE__', $Zone) + # write the runner on the VM (base64 to avoid quoting hell), then launch detached + $b64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($runner)) + # Run the runner as the SSH user (NOT sudo) so ~ resolves to the user's home + # where the bundle was scp'd; the runner's own docker/gcloud calls use sudo. + GcloudSsh "echo $b64 | base64 -d > ~/vlbench-runner.sh && chmod +x ~/vlbench-runner.sh && setsid bash ~/vlbench-runner.sh /dev/null 2>&1 & echo LAUNCHED" + Write-Host "" + Write-Host ">> LAUNCHED autonomous run on $Name." + Write-Host " results -> $Bucket/$Name/" + Write-Host " progress -> gcloud compute ssh $Name --zone=$Zone --tunnel-through-iap --command 'tail -f ~/vlbench-run.log'" + Remove-Item -Force $tgzName -ErrorAction SilentlyContinue +} +catch { + Write-Host "ERROR: $_" + Write-Host ">> deleting VM $Name (launch failed)" + gcloud compute instances delete $Name --project=$Project --zone=$Zone --quiet + throw +} diff --git a/deploy/gcp/startup-autonomous.sh b/deploy/gcp/startup-autonomous.sh new file mode 100644 index 0000000..fa8076e --- /dev/null +++ b/deploy/gcp/startup-autonomous.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# All-in-one autonomous VM runner, driven entirely by instance metadata — no +# SSH/IAP needed (so it works cleanly when launched from a GitHub Actions +# workflow). On boot it: installs Docker + gcloud, pulls the staged bundle + +# .env from GCS, builds the image, downloads FinanceBench, runs the benchmark, +# uploads results AND this log to GCS, then powers off (the instance's +# max-run-duration is the deletion backstop). +# +# Required instance metadata attributes: +# vlbench-staging gs://bucket/_staging// (bundle.tgz + env live here) +# vlbench-results gs://bucket// (results + run.log go here) +# vlbench-config configs/financebench_threeway.yaml +set -uxo pipefail +LOG=/var/log/vlbench-run.log +exec > >(tee -a "$LOG") 2>&1 + +meta() { curl -fsS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$1"; } +STAGING_URI=$(meta vlbench-staging) +RESULTS_URI=$(meta vlbench-results) +CONFIG=$(meta vlbench-config) +LIMIT=$(meta vlbench-limit || echo 0) # 0 = use the config's own limit + +# Always push the log (for post-mortem) and stop the VM when we exit, however we +# exit. Deletion is handled by the instance's --max-run-duration backstop. +cleanup() { + rc=$? + echo "EXIT rc=$rc $(date -u)" + gcloud storage cp "$LOG" "$RESULTS_URI/run.log" || true + poweroff +} +trap cleanup EXIT + +echo "START $(date -u) staging=$STAGING_URI results=$RESULTS_URI config=$CONFIG" + +# ── install Docker + gcloud (same approach as startup-script.sh) ── +if ! command -v docker >/dev/null 2>&1; then + curl -fsSL https://get.docker.com | sh +fi +if ! command -v gcloud >/dev/null 2>&1; then + snap install google-cloud-cli --classic +fi + +set -e +mkdir -p /opt/vlbench && cd /opt/vlbench +gcloud storage cp "$STAGING_URI/bundle.tgz" bundle.tgz +gcloud storage cp "$STAGING_URI/env" .env.staged +tar -xzf bundle.tgz +cp .env.staged .env + +LIMARG="" +if [ "${LIMIT:-0}" != "0" ]; then LIMARG="--limit $LIMIT"; fi + +docker compose build +docker compose run --rm --entrypoint python bench scripts/download_financebench.py +docker compose run --rm bench run --config "$CONFIG" $LIMARG --out /results + +echo "RUN DONE $(date -u); uploading results to $RESULTS_URI/" +gcloud storage cp -r results/* "$RESULTS_URI/" +echo "UPLOADED $(date -u)" +# cleanup() trap uploads the log and powers off diff --git a/deploy/load_secrets.py b/deploy/load_secrets.py new file mode 100644 index 0000000..363b8b9 --- /dev/null +++ b/deploy/load_secrets.py @@ -0,0 +1,101 @@ +"""Materialize a .env for the benchmark from Google Cloud Secret Manager. + +The Vectorless deployment keeps its config in a single Secret Manager secret +(`server-config`, the engine's YAML), so the API keys live inside it rather than +as standalone secrets. This reads that secret with the caller's gcloud identity +(works locally and on a GCE VM via its service account) and extracts exactly what +the benchmark needs into a .env — never printing the values. + +Mapping (server-config YAML -> env): + auth.api_key -> VECTORLESS_API_KEY (engine bearer key) + engine.llm.gemini.api_key -> GEMINI_API_KEY + GOOGLE_API_KEY (baselines via google-genai) + --base-url -> VECTORLESS_BASE_URL (the engine's Cloud Run URL) + +Usage: + python deploy/load_secrets.py \ + --project project-03250746-ec5b-4198-990 \ + --base-url https://vectorless-server-2rzh3kctga-uc.a.run.app \ + --out .env +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from typing import Any, Optional + +import yaml + + +def _access_secret(project: str, secret: str) -> str: + """Fetch the latest version of a secret via the gcloud CLI (uses ambient auth). + + `gcloud secrets versions access` returns the decoded payload on stdout.""" + res = subprocess.run( + ["gcloud", "secrets", "versions", "access", "latest", + f"--secret={secret}", f"--project={project}"], + capture_output=True, text=True, shell=(sys.platform == "win32"), + ) + if res.returncode != 0: + raise SystemExit(f"failed to read secret {secret!r}: {res.stderr.strip()}") + return res.stdout + + +def _dig(d: Any, *path: str) -> Optional[Any]: + cur = d + for key in path: + if not isinstance(cur, dict) or key not in cur: + return None + cur = cur[key] + return cur + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--project", required=True) + ap.add_argument("--secret", default="server-config") + ap.add_argument("--base-url", required=True, help="engine Cloud Run URL") + ap.add_argument("--out", default=".env") + args = ap.parse_args() + + cfg = yaml.safe_load(_access_secret(args.project, args.secret)) or {} + + vl_key = _dig(cfg, "auth", "api_key") + gem_key = _dig(cfg, "engine", "llm", "gemini", "api_key") + # OpenAI key powers the vector_rag / full_context / judge baselines. The + # engine itself may run on a different provider (e.g. GLM via the + # Anthropic-compatible driver), so this is best-effort: pull it from the + # engine LLM config or a top-level openai block if present. + openai_key = _dig(cfg, "engine", "llm", "openai", "api_key") or _dig(cfg, "openai", "api_key") + if not vl_key: + raise SystemExit("server.auth.api_key not found in secret") + if not gem_key: + print("WARN: llm.gemini.api_key not found — Gemini baselines will fail", file=sys.stderr) + if not openai_key: + print("WARN: openai.api_key not found in secret — set OPENAI_API_KEY yourself " + "for the vector_rag/full_context/judge baselines", file=sys.stderr) + + lines = [ + "# generated by deploy/load_secrets.py — do not commit", + f"VECTORLESS_BASE_URL={args.base_url}", + f"VECTORLESS_API_KEY={vl_key}", + ] + if gem_key: + lines += [f"GEMINI_API_KEY={gem_key}", f"GOOGLE_API_KEY={gem_key}"] + if openai_key: + lines += [f"OPENAI_API_KEY={openai_key}"] + + with open(args.out, "w", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + # report what was written WITHOUT exposing values + wrote = ["VECTORLESS_BASE_URL", "VECTORLESS_API_KEY"] + ( + ["GEMINI_API_KEY", "GOOGLE_API_KEY"] if gem_key else [] + ) + (["OPENAI_API_KEY"] if openai_key else []) + print(f"wrote {args.out} with: {', '.join(wrote)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 5abe188..39116a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,8 +19,9 @@ dependencies = [ [project.optional-dependencies] # token counting + the LLM-using systems (full_context, judge) llm = ["tiktoken>=0.7", "openai>=1.40", "anthropic>=0.39"] -# the vector-RAG baseline -vector = ["openai>=1.40", "psycopg[binary]>=3.1", "pgvector>=0.3", "tiktoken>=0.7"] +# the vector-RAG baseline. sentence-transformers powers the local +# (no-API-key) embedding option, e.g. embedding_model="st:BAAI/bge-small-en-v1.5". +vector = ["openai>=1.40", "psycopg[binary]>=3.1", "pgvector>=0.3", "tiktoken>=0.7", "sentence-transformers>=2.7"] # the BM25 baseline bm25 = ["rank-bm25>=0.2.2"] # the system under test @@ -33,6 +34,7 @@ viz = ["matplotlib>=3.8"] all = [ "tiktoken>=0.7", "openai>=1.40", "anthropic>=0.39", "psycopg[binary]>=3.1", "pgvector>=0.3", "rank-bm25>=0.2.2", + "sentence-transformers>=2.7", "vectorless-sdk>=0.1", "datasets>=2.19", "pypdf>=4.0", "requests>=2.31", "matplotlib>=3.8", ] diff --git a/src/vectorless_bench/retrievers/vector_rag.py b/src/vectorless_bench/retrievers/vector_rag.py index 37d9075..8357186 100644 --- a/src/vectorless_bench/retrievers/vector_rag.py +++ b/src/vectorless_bench/retrievers/vector_rag.py @@ -57,6 +57,12 @@ def __init__( self._conn = None self._table = "vlbench_chunks" self._openai = None + self._st = None + # A "st:" embedding_model selects a local + # sentence-transformers embedder (free, deterministic, no API key) — + # the no-OpenAI path for a fully self-hosted vector-RAG baseline. + self._is_local = embedding_model.startswith("st:") + self._st_model_id = embedding_model[3:] if self._is_local else "" # -- embeddings -------------------------------------------------------- def _client(self): @@ -66,14 +72,35 @@ def _client(self): self._openai = OpenAI() return self._openai - def _embed(self, texts: Sequence[str]) -> List[List[float]]: + def _st_model(self): + if self._st is None: + from sentence_transformers import SentenceTransformer # type: ignore + + self._st = SentenceTransformer(self._st_model_id) + # keep self.dim in sync so the pgvector column matches + self.dim = int(self._st.get_sentence_embedding_dimension()) + return self._st + + def _encode(self, texts: Sequence[str]) -> List[List[float]]: + """Embed texts via either a local sentence-transformers model or the + OpenAI-compatible client, depending on embedding_model.""" + if self._is_local: + vecs = self._st_model().encode( + list(texts), normalize_embeddings=True, show_progress_bar=False + ) + return [list(map(float, v)) for v in vecs] resp = self._client().embeddings.create( model=self.embedding_model, input=list(texts) ) + return [d.embedding for d in resp.data] + + def _embed(self, texts: Sequence[str]) -> List[List[float]]: + out = self._encode(texts) toks = sum(count_tokens(t, self.embedding_model) for t in texts) self.setup_usage.embedding_tokens += toks + # local embeddings are free; compute_embedding returns 0 for unpriced self.setup_usage.cost_usd += compute_embedding(self.embedding_model, toks) - return [d.embedding for d in resp.data] + return out # -- lifecycle --------------------------------------------------------- def setup(self, corpus: List[Doc]) -> None: @@ -172,8 +199,7 @@ def retrieve(self, question: Question, k: int, cold: bool = True) -> RetrievalRe def _embed_query(self, q: str) -> List[float]: # query embedding cost is tiny; don't fold it into ingest usage - resp = self._client().embeddings.create(model=self.embedding_model, input=[q]) - return resp.data[0].embedding + return self._encode([q])[0] def _pg_query(self, qvec, doc_id, k): # the cosine operator (<=>) appears in both SELECT and ORDER BY, so the