diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3816d5c..c4eb3e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ jobs: - run: bun install --frozen-lockfile - run: bun run typecheck - run: bun run test + - run: bun run test:e2e node: runs-on: ubuntu-latest strategy: diff --git a/CHANGELOG.md b/CHANGELOG.md index fa175dc..d989bc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,217 +5,48 @@ All notable changes to `@corbits/memory` are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.2.0] — 2026-09-25 ### Changed -- `@intx/*`, `drizzle-orm`, `hono`, `hono-openapi` and `postgres` are peer - dependencies; the host supplies them. `engines` is removed. +- **Breaking:** `@intx/*`, `drizzle-orm`, `hono`, `hono-openapi` and + `postgres` are peer dependencies; the host supplies them. `engines` is + removed. +- **Breaking:** `createMemoryRoutes({ memory, requireGrant, callerResolver? })` + returns the memory routes as a `Hono` sub-app with paths relative + to its mount point; hosts mount it at `/api/tenants/:tenantId/memory`. It + replaces `createMemory({ app, callerResolver })` and `registerMemoryRoutes`, + and `RouteDeps` no longer carries `grants`. `createMemory` only builds the + plane. +- **Breaking:** the package root exports only the public API. Internal + services and helpers (transform, retention, feed, share materialization, + corroboration, embed model registry, degrade metrics, FTS helpers), the test + fakes, and `resolveGrantConfig` are no longer exported. The distiller stays + at `@corbits/memory/distiller` and migrations at + `@corbits/memory/migrations`. +- **Breaking:** `runMemoryMigrations(config, { schema, ftsLanguage })` takes + the same `DBConfig` as Interchange `runMigrations` instead of a database + URL. `schema` names the host schema holding Interchange's `tenant` and + `principal` tables (the value passed to `runMigrations`, e.g. `"public"`); + memory's tables stay in the `memory` schema. `ftsLanguage` is required, and + the runner no longer reads `FTS_LANGUAGE` from the environment or accepts a + `log` option. +- Every migration file is idempotent and replayed on each run, with a 5 s + `lock_timeout`. The `memory._migrations` ledger is dropped. - `MEMORY_GRANT_REQUIREMENTS` is read from `package.json` `interchange.grantRequirements`, now the only declaration. `MEMORY_CAPABILITY_IDS` is typed `string[]`. -- `createMemoryRoutes({ memory, requireGrant, callerResolver? })` returns the - memory routes as a `Hono` sub-app with paths relative to its - mount point; hosts mount it at `/api/tenants/:tenantId/memory`. It replaces - `createMemory({ app })` and `registerMemoryRoutes`, and `RouteDeps` no - longer carries `grants`. `createMemory` only builds the plane. -- The package root exports only the public API. Internal services and - helpers (transform, retention, feed, share materialization, corroboration, - embed model registry, degrade metrics, FTS helpers), the test fakes, and - `resolveGrantConfig` are no longer exported. The distiller stays at - `@corbits/memory/distiller` and migrations at `@corbits/memory/migrations`. -- `runMemoryMigrations(config, { schema, ftsLanguage })` takes the same - `DBConfig` as Interchange `runMigrations` instead of a database URL. - `schema` names the host schema holding Interchange's `tenant` and - `principal` tables (the value passed to `runMigrations`, e.g. `"public"`); - memory's tables stay in the `memory` schema. `ftsLanguage` is required, and - the runner no longer reads `FTS_LANGUAGE` from the environment or accepts a `log` option. - Every migration file is idempotent and replayed on each run. The - `memory._migrations` ledger is dropped. - -### Added - -- Retention HTTP routes (CL-6288): `POST …/memory/documents/:documentId/forget` - (tombstone, grant `memory:forget`), `POST …/memory/documents/:documentId/purge` - (hard delete, grant `memory:purge`), and - `POST …/memory/versions/:versionId/retention-class` (grant `memory:forget`). - Forget and purge are separate routes with separate grant actions — never one - route with a boolean flag — and both are refused with 403 unless the caller - is the document/version's creator, independent of any share grant that lets - them merely see it. `sweepEphemeral` stays off the HTTP surface (maintenance - sweep, not a user action); a host schedules it on its own cron against the - in-process `Memory`. New `memory:forget` / `memory:purge` grant requirements - (`source: "creator"`) and `capabilityIdsForSurface()` so distiller/tools - installs no longer pick up routes-only capabilities by accident. -- `RouteDeps.callerResolver` — an - optional host-supplied resolver from a request to a `{ tenantId, - principalId }` scope, for a caller that never goes through the host's - tenant-session middleware (e.g. a workflow-run child authenticating with - its own sidecar bearer token). Unset by default: every route still reads - identity from `c.get("principal")` exactly as before. When set, the - resolved identity is seated as the request's principal/tenant ahead of - `grantGuard`, so the same `requireGrant` authorization path applies to a - machine caller — never a separate, weaker one. Identity from the resolver - always wins over anything a request body claims. The resolver's return - value is parsed with arktype (non-empty `tenantId`/`principalId`); a - resolver returning a malformed identity is rejected with `500` (a host - bug), never seated as a garbage scope. - -### Fixed - -- `loadMemoryConfig` / `EngineConfig.embed` no longer requires an embed - endpoint: a host with a pgvector Postgres and no embed endpoint now - constructs and serves `add` + lexical `search`. Dense retrieval is skipped - (not attempted-and-failed) and `search` reports - `degraded: ["dense_unavailable", "lexical_only"]` so the state stays - observable (CL-6287). **Migration note:** `dense_unavailable` is now also - emitted on every search for a deliberately-unconfigured engine, not only on - a transient dense-retrieval failure — a host with an existing alert rule - keyed on `dense_unavailable` alone should also check for `lexical_only` in - the same `degraded` array to distinguish "opted into lexical-only" from an - actual regression. -- `add`'s `degraded` is now a reason array (`["embed_unavailable"]` and/or - `["embed_unavailable", "lexical_only"]`), matching `search`'s shape — - previously a bare boolean, which made it impossible to write one - "is this response degraded" check across both verbs (CL-6287). **Breaking - if a host coded against the boolean:** `degraded: true` is now - `degraded: [...]`; check array presence/length instead of truthiness (both - are still falsy/omitted when the document captured cleanly). -- `Memory.capabilities.embeddingsConfigured` (and the underlying - `DocumentStore.capabilities`) let a host learn recall is lexical-only at - construction time, without issuing a search first (CL-6287). -- Feed `nextCursor` advances past the examined raw page after grant-tag - post-filter (a fully denied page no longer stalls the consumer forever). -- Distiller default system prompt uses the configured `agentId` for - `exclude_generator` / `generator_agent_id` (not only the package default id). - -### Changed - -- **Breaking:** `createResidentDistiller` runs on a `mail` trigger (CL-8798). - `CreateResidentDistillerOpts.cron` is replaced by a required `mailTo`, the - address the workflow's run is triggered at, and - `RESIDENT_DISTILLER_CRON_DEFAULT` is removed. The `schedule` trigger was - reserved with nothing behind it in `@intx/workflow`, so the distiller never - ticked, and 0.4.0 rejects it at `defineWorkflow`. Ticking is the host's job: - mail `mailTo` on a schedule, e.g. from `@corbits/cron`. -- `@intx/*` dependencies move to 0.4.0 (CL-8797). -- **Product narrative:** default path is **add → ingest elements → process** - (one host pipeline). Pull feed + `createResidentDistiller` are optional - multi-writer / backfill process helpers, not the primary ingest story. - See `PRODUCT.md`, `docs/DISTILLER.md`, `docs/FEED.md`. -- **Breaking:** Postgres schema renamed from `knowledge` to **`memory`**. Fresh - installs only — drop/recreate the old schema (or rename) on existing DBs. - Citation `open.type` is now `"memory"`. -- **Breaking:** env var is `DATABASE_URL` (was `KNOWLEDGE_DATABASE_URL`); pass - `memory.databaseUrl` on config as an alternative. No deprecated alias. -- `add` (plane + HTTP) returns `{ documentId, versionId }` so share-grant audit - and provenance can name the version that carried the write. -- Interchange `defineTool` factories at `@corbits/memory/tools` (`memory_add`, - `memory_search`, `memory_list`) — HTTP clients for mounted hub routes with - install env `memoryBaseUrl` / `memoryTenantId` / `memoryAuthToken`. Declared - via `package.json` `interchange.tools` and `exports["./tools"]`. - -### Previously - -- **Breaking:** package and public surface renamed from `@corbits/knowledge-engine` - to `@corbits/memory`. Public APIs: `createMemory` (optional `app` registers HTTP), - `registerMemoryRoutes`. - - `createMemory`, `loadMemoryConfig`, `runMemoryMigrations`, `Memory`, - `MemoryConfig`, `MemoryError`. HTTP paths are under - `/api/tenants/:tenantId/memory/`; grants are - `memory:add` / `memory:search`; access tags use `memory.owner:` / `memory.tenant:` - / `memory.space:`. Postgres schema is `memory`. -- **Breaking:** config reads `DATABASE_URL` (was `KNOWLEDGE_DATABASE_URL`) for - the engine's own pgvector Postgres connection. -- **Breaking:** memory plane surface is `add` / `search` / `list` with - `principalId` + `tenantId` only. Inference is host-owned (no answer endpoint - or personal-memory side-channel on the plane). - -- **Breaking:** HTTP routes are `POST /api/tenants/:tenantId/memory/add`, - `POST /api/tenants/:tenantId/memory/search`, - `GET /api/tenants/:tenantId/memory/list` (inherits hub `resolveTenant`). - Old unscoped `/api/memory/*` paths are not mounted. -- **Breaking:** grant actions are `add` and `search` (was `capture` / `find` / - knowledge `search`). `list` uses the `search` grant. Capability resource is - `memory`. Document-tag checks use action `search`. -- **Breaking:** `add` returns `{ documentId }`; search body uses `limit` (not `k`); - search wire uses `items` (not `hits`). -- **Breaking:** document access is Interchange **grant tags** (`accessTags` + - creator-always + host `GrantStore`), not the visibility-mode / block-list - mini-ACL. Share sugar only mints tags. See `docs/AUTHZ-DOCUMENT-ACCESS.md`. -- **Breaking:** Postgres baseline is two files (`0001_extensions` + - `0002_memory_baseline`) with `access_tags` and no `visibility_*` columns. - Fresh installs only — drop/recreate the `memory` schema on existing DBs. -- **Breaking:** `grantStore` + `conditionRegistry` are top-level `createMemory` - options (no nested `grants: { … }`). - -### Added - -- Optional `TextExtractor` + `file` XOR `content` on `add` -- `share` sugar on `add` (maps to access tags; principals also materialize - grants when the host grant store is writable); `grantsMaterialized` on the - add result when peers were requested -- `access_tags` on `memory.document` (baseline schema) -- Claim-bearing schema, temporal model, transform/replay, share grants - (resident memory distillation foundation — CL-5865/5866/5872/5873) -- Living relevancy: corroboration factor from supports/contradicts edges; - strong evidence gate (CL-5867). See `docs/RELEVANCY.md` -- Capture feed: `memory.feed` + `GET .../memory/feed` with `feed_seq` cursor - (CL-5868). See `docs/FEED.md` -- **Wire attribution (CL-5870):** search hits carry additive `attribution` - (versionId, provenance, source/temporal class, createdByKind, - generatorAgentId, occurredAt/validUntil, corroboration counts, derivedFrom) -- **Retention (CL-5871):** `retention_class` on versions; plane APIs - `deprecateVersion` / `tombstoneDocument` / `hardDeleteDocument` / - `sweepEphemeral` / `setRetentionClass`; `includeDeprecated` on search. - See `docs/RETENTION.md` -- **Resident distiller (CL-5869):** `@corbits/memory/distiller` — - `createResidentDistiller({ inference })` schedule workflow + `runDistillTick` - + `buildDistilledClaim`. Claim-aware `add` (generator_agent_id, provenance, - derived_from, …). `memory_feed` tool. Feed entries include `accessTags`. - See `docs/DISTILLER.md` - -### Removed - -- Host-injected generate path on the plane (use host inference + `add` / `search`) - -## [0.1.2] — 2026-07-31 - -### Added -- Public `createMemory` export for out-of-band capture and search (CLI seeders, batch ingesters, tests) without mounting HTTP routes (`#8`) - -### Fixed - -- Fail-closed document ACL: withhold hits whose `acl_block` is unreadable or non-string (`#9`) -- Timeline titles use the same document ACL post-filter as search (`#12`) -- Return 401 (not 500) when the host has not resolved a principal (`#7`) -- ACL wiring tests mock `sql.unsafe` for FTS verification so the suite stays green under the search path - -## [0.1.1] — 2026-07-31 - -### Added - -- Halfvec expression indexes for embedding models above 2000 dimensions (up to 4000); dense search ORDER BY uses the same expression as the index (`#1`) -- Configurable lexical FTS language (`FTS_LANGUAGE` / `EngineConfig.ftsLanguage`) with parse, catalog verify, and rebuild recipe; public `DEFAULT_FTS_LANGUAGE` export (`#4`) -- Named chunk FK with `ON DELETE CASCADE` and composite tenant index on per-model embedding tables created at activation (`#3`) - -### Changed - -- Dense search scales `hnsw.ef_search` to the overfetch limit (floor 40) and probes `hnsw.iterative_scan = relaxed_order` once per process when available (`#2`) -- Package install docs point at `@corbits/memory` (`#6`) - -### Fixed -- Rerank document truncation to fit cross-encoder token limits (`#10`) -- Degrade metrics: surface full rerank failure detail and count DegradeFlags so silent degradation is visible (`#11`) +### Upgrading from 0.1.0 -## [0.1.0] — 2026-07 +- Call the new `runMemoryMigrations` once; it upgrades a 0.1.0 database in + place with no data loss. +- After the upgrade, do not run 0.1.0 against the database: its ledger is + gone, so there is no downgrade. -### Added +## [0.1.0] — 2026-09-25 -- Initial cut: mountable memory capture + search SDK for Interchange hubs +First release on npm. -[0.1.2]: https://github.com/corbitsdev/corbits-memory/compare/v0.1.1...v0.1.2 -[0.1.1]: https://github.com/corbitsdev/corbits-memory/compare/v0.1.0...v0.1.1 -[0.1.0]: https://github.com/corbitsdev/corbits-memory/releases/tag/v0.1.0 +[0.2.0]: https://github.com/corbitsdev/corbits-memory/releases/tag/v0.2.0 +[0.1.0]: https://www.npmjs.com/package/@corbits/memory/v/0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e510af5..ebf7ec0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,11 +22,11 @@ Postgres. See `IMPLEMENTATION.md` for how the pieces fit together. ## Running the tests ```bash -bun run typecheck && bun run test +bun run typecheck && bun run test && bun run test:e2e ``` -- `bun run test` runs the unit suite in `src/` and the end-to-end suite in - `tests/`. The end-to-end tests drive the mounted routes and migrations +- `bun run test` runs the unit suite in `src/`; `bun run test:e2e` runs the + end-to-end suite in `e2e/`. The end-to-end tests drive the mounted routes and migrations against a real pgvector Postgres: set `TEST_DATABASE_URL` to a server the tests can create and drop databases on (for `docker compose up -d`, `postgres://memory:memory-dev-password@localhost:5434/memory`). Each suite @@ -53,18 +53,16 @@ numbered file, because a guarded `ADD CONSTRAINT` keeps the old definition. changed; link any relevant issue. - Make sure `bun run typecheck && bun run test` pass before requesting review. -## Commit messages - -- Present tense, imperative mood: "Add X", "Fix Y", "Harden Z" — not "Added" - or "Fixes". -- No issue-tracker ticket references (e.g. `CL-1234`) in commit messages, - code, or comments — commit messages should be self-explanatory without an - external ticket. -- One logical change per commit where practical; a commit message describes - the change, not the task that produced it. - ## Contributor License Agreement Contributions require agreeing to the project's CLA — see `CLA.md`. The CLA bot will comment on your first PR with instructions if you haven't signed yet. + +## Commit messages + +Commit subjects and PR titles follow [Conventional Commits](https://www.conventionalcommits.org): `feat`, `fix`, `refactor`, `test`, `docs`, `build`, `ci`, `perf`, and `chore(release): x.y.z` for releases. +Add `!` only for public API breaks: removed or renamed exports, changed signatures, newly required params. Peer and dependency range changes are `build(deps):` with no `!`. +Keep subjects imperative, lowercase after the colon, 72 characters or less, and free of ticket IDs. +No ticket IDs in code or comments either. One logical change per commit where practical; describe the change, not the task that produced it. +Every PR links its issue with a `Closes ` line in the PR body. diff --git a/bun.lock b/bun.lock index 9976731..63d603e 100644 --- a/bun.lock +++ b/bun.lock @@ -8,6 +8,7 @@ "arktype": "^2.1.29", }, "devDependencies": { + "@corbits/memory-0.1.0": "npm:@corbits/memory@0.1.0", "@intx/agent": "0.4.0", "@intx/authz": "0.4.0", "@intx/db": "0.4.0", @@ -60,6 +61,8 @@ "@better-fetch/fetch": ["@better-fetch/fetch@1.3.1", "", {}, "sha512-ABkD1WhyfPZprKRQI3bhATjeiFuNWC9PXhfGWqL+sg/gKrM977oFrYkdb4msM3hgUGonr7KlOsOFT5TU2rht9g=="], + "@corbits/memory-0.1.0": ["@corbits/memory@0.1.0", "", { "dependencies": { "@intx/agent": "^0.4.0", "@intx/authz": "^0.4.0", "@intx/hub-api": "^0.4.0", "@intx/log": "^0.4.0", "@intx/types": "^0.4.0", "@intx/workflow": "^0.4.0", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", "hono-openapi": "^1.3.1", "postgres": "^3.4.7" } }, "sha512-+fzKplQ3VPCnhUK+WLo5EDKAwJSi61TX3yySeKBxX3wJTTC8BhHun5Ga/nn6Shxd3zShN1DklWd1koCfgwOGLg=="], + "@gar/promise-retry": ["@gar/promise-retry@1.0.3", "", {}, "sha512-GmzA9ckNokPypTg10pgpeHNQe7ph+iIKKmhKu3Ob9ANkswreCx7R3cKmY781K8QK3AqVL3xVh9A42JvIAbkkSA=="], "@hono/standard-validator": ["@hono/standard-validator@0.2.3", "", { "peerDependencies": { "@standard-schema/spec": "^1.0.0", "hono": ">=3.9.0" } }, "sha512-bp9vHu6Va6SfMHC3D4ZLBbT/woi+AZ9CRdTXQu3kLJuLh2W/Gb9UO4hijS+BQAGFXi4EGpXdetxpzwTAawSVeg=="], diff --git a/docs/RETENTION.md b/docs/RETENTION.md index b878420..b722f19 100644 --- a/docs/RETENTION.md +++ b/docs/RETENTION.md @@ -64,7 +64,7 @@ creator (`created_by_principal_id` — the document's first version for independent of any share grant. A peer who can search a shared document gets 403 on `forget`/`purge`/`retention-class` for it. See `src/services/retention-ownership.ts` and the ownership tests in -`tests/grants.test.ts` / `tests/caller-resolver.test.ts`. +`e2e/grants.test.ts` / `e2e/caller-resolver.test.ts`. **`sweepEphemeral` stays off the HTTP surface.** It is a maintenance sweep — "deprecate every ephemeral version past its TTL for this tenant" — not diff --git a/tests/add-search.test.ts b/e2e/add-search.test.ts similarity index 99% rename from tests/add-search.test.ts rename to e2e/add-search.test.ts index bc31908..182a1eb 100644 --- a/tests/add-search.test.ts +++ b/e2e/add-search.test.ts @@ -12,7 +12,7 @@ import { seedPrincipal, testDatabaseUrl, type TestDb, -} from "./lib/db-harness.ts"; +} from "./helpers.ts"; describe.skipIf(testDatabaseUrl() === undefined)("add and search", () => { let db: TestDb; diff --git a/tests/caller-resolver.test.ts b/e2e/caller-resolver.test.ts similarity index 99% rename from tests/caller-resolver.test.ts rename to e2e/caller-resolver.test.ts index 896f871..dec784e 100644 --- a/tests/caller-resolver.test.ts +++ b/e2e/caller-resolver.test.ts @@ -15,7 +15,7 @@ import { seedPrincipal, testDatabaseUrl, type TestDb, -} from "./lib/db-harness.ts"; +} from "./helpers.ts"; // A machine caller (e.g. a workflow run) never passes the host's session // middleware; the host's callerResolver names its tenant and principal. diff --git a/tests/context-rows.test.ts b/e2e/context-rows.test.ts similarity index 98% rename from tests/context-rows.test.ts rename to e2e/context-rows.test.ts index 22dc782..7ae30e9 100644 --- a/tests/context-rows.test.ts +++ b/e2e/context-rows.test.ts @@ -12,7 +12,7 @@ import { seedPrincipal, testDatabaseUrl, type TestDb, -} from "./lib/db-harness.ts"; +} from "./helpers.ts"; describe.skipIf(testDatabaseUrl() === undefined)("synthesized context rows", () => { let db: TestDb; diff --git a/tests/distill-tick.test.ts b/e2e/distill-tick.test.ts similarity index 99% rename from tests/distill-tick.test.ts rename to e2e/distill-tick.test.ts index 891a563..b844758 100644 --- a/tests/distill-tick.test.ts +++ b/e2e/distill-tick.test.ts @@ -12,7 +12,7 @@ import { seedPrincipal, testDatabaseUrl, type TestDb, -} from "./lib/db-harness.ts"; +} from "./helpers.ts"; describe.skipIf(testDatabaseUrl() === undefined)("distill tick", () => { let db: TestDb; diff --git a/tests/embed-client.test.ts b/e2e/embed-client.test.ts similarity index 98% rename from tests/embed-client.test.ts rename to e2e/embed-client.test.ts index 6c9abd5..8fb4826 100644 --- a/tests/embed-client.test.ts +++ b/e2e/embed-client.test.ts @@ -6,7 +6,7 @@ import { embedTexts, probeEmbedDims, } from "../src/core/embed-client.ts"; -import { startHttpStub } from "./lib/http-stub.ts"; +import { startHttpStub } from "./helpers.ts"; const stub = startHttpStub(); afterAll(() => stub.stop()); diff --git a/tests/grants.test.ts b/e2e/grants.test.ts similarity index 99% rename from tests/grants.test.ts rename to e2e/grants.test.ts index 9c6d68e..bfdb095 100644 --- a/tests/grants.test.ts +++ b/e2e/grants.test.ts @@ -12,7 +12,7 @@ import { seedPrincipal, testDatabaseUrl, type TestDb, -} from "./lib/db-harness.ts"; +} from "./helpers.ts"; describe.skipIf(testDatabaseUrl() === undefined)("grants, forget and purge", () => { let db: TestDb; diff --git a/tests/lib/db-harness.ts b/e2e/helpers.ts similarity index 78% rename from tests/lib/db-harness.ts rename to e2e/helpers.ts index bb1cc2b..ca0aa62 100644 --- a/tests/lib/db-harness.ts +++ b/e2e/helpers.ts @@ -1,4 +1,4 @@ -// Real-Postgres harness for the tests/ suite. Each suite gets a fresh +// Real-Postgres harness for the e2e/ suites. Each suite gets a fresh // database on the server named by TEST_DATABASE_URL (any pgvector Postgres // whose user can create databases, such as compose.yml's), with Interchange's control-plane tables and the memory // migrations applied; `close` drops it. @@ -9,10 +9,10 @@ import { createRequireGrant, type TenantEnv } from "@intx/hub-api"; import { Hono } from "hono"; import postgres from "postgres"; -import { createMemory, type Memory } from "../../src/memory.ts"; -import type { MemoryConfig } from "../../src/mount-config.ts"; -import { runMemoryMigrations } from "../../src/migrations.ts"; -import { createMemoryRoutes } from "../../src/routes/mount.ts"; +import { createMemory, type Memory } from "../src/memory.ts"; +import type { MemoryConfig } from "../src/mount-config.ts"; +import { runMemoryMigrations } from "../src/migrations.ts"; +import { createMemoryRoutes } from "../src/routes/mount.ts"; const FTS_LANGUAGE = "english"; @@ -49,7 +49,7 @@ export type TestDb = { export async function createEmptyDb(): Promise { const serverUrl = testDatabaseUrl(); if (serverUrl === undefined) { - throw new Error("TEST_DATABASE_URL is required for the tests/ suite"); + throw new Error("TEST_DATABASE_URL is required for the e2e/ suites"); } const url = new URL(serverUrl); const database = `memory_test_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`; @@ -205,3 +205,50 @@ export function createTestApp(opts: { ); return app; } + +// A local HTTP server standing in for an embed or rerank endpoint. Each test +// sets `reply`; every request is recorded with its path, auth header and +// JSON body (both clients only send JSON). + +export type StubRequest = { + path: string; + authorization: string | null; + body: unknown; +}; + +export type HttpStub = { + url: string; + requests: StubRequest[]; + reply: (req: StubRequest) => Response | Promise; + reset: () => void; + stop: () => void; +}; + +const unconfigured = () => new Response("no reply configured", { status: 500 }); + +export function startHttpStub(): HttpStub { + const stub: HttpStub = { + url: "", + requests: [], + reply: unconfigured, + reset: () => { + stub.requests.length = 0; + stub.reply = unconfigured; + }, + stop: () => server.stop(true), + }; + const server = Bun.serve({ + port: 0, + async fetch(req) { + const recorded: StubRequest = { + path: new URL(req.url).pathname, + authorization: req.headers.get("authorization"), + body: await req.json(), + }; + stub.requests.push(recorded); + return stub.reply(recorded); + }, + }); + stub.url = server.url.href.replace(/\/$/, ""); + return stub; +} diff --git a/tests/migrations.test.ts b/e2e/migrations.test.ts similarity index 98% rename from tests/migrations.test.ts rename to e2e/migrations.test.ts index 7cdfcb5..4424fe9 100644 --- a/tests/migrations.test.ts +++ b/e2e/migrations.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { runMigrations } from "@intx/db"; import { runMemoryMigrations } from "../src/migrations.ts"; -import { createEmptyDb, testDatabaseUrl, type TestDb } from "./lib/db-harness.ts"; +import { createEmptyDb, testDatabaseUrl, type TestDb } from "./helpers.ts"; const options = { schema: "public", ftsLanguage: "english" }; diff --git a/tests/plane.test.ts b/e2e/plane.test.ts similarity index 99% rename from tests/plane.test.ts rename to e2e/plane.test.ts index 5162215..aaeb617 100644 --- a/tests/plane.test.ts +++ b/e2e/plane.test.ts @@ -15,7 +15,7 @@ import { testDatabaseUrl, testMemoryConfig, type TestDb, -} from "./lib/db-harness.ts"; +} from "./helpers.ts"; async function rejection(run: Promise): Promise { const err = await run.then( diff --git a/tests/rerank-client.test.ts b/e2e/rerank-client.test.ts similarity index 99% rename from tests/rerank-client.test.ts rename to e2e/rerank-client.test.ts index ab65075..7dd7827 100644 --- a/tests/rerank-client.test.ts +++ b/e2e/rerank-client.test.ts @@ -12,7 +12,7 @@ import { rerankDocuments, validateRerankConfig, } from "../src/core/rerank-client.ts"; -import { startHttpStub } from "./lib/http-stub.ts"; +import { startHttpStub } from "./helpers.ts"; // .env.example ships RERANK_MODEL=bge-reranker-base with RERANK_MAX_DOC_CHARS // unset; that combination must validate. diff --git a/e2e/upgrade-from-0.1.0.test.ts b/e2e/upgrade-from-0.1.0.test.ts new file mode 100644 index 0000000..cfcb21d --- /dev/null +++ b/e2e/upgrade-from-0.1.0.test.ts @@ -0,0 +1,118 @@ +// A database migrated and written by the published 0.1.0 package upgrades in +// place under this version's runMemoryMigrations with no data loss. +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { createInMemoryGrantStore } from "@intx/authz"; +import { runMigrations } from "@intx/db"; +import * as v010 from "@corbits/memory-0.1.0"; + +import { runMemoryMigrations } from "../src/migrations.ts"; +import { + allow, + createEmptyDb, + createTestMemory, + seedPrincipal, + testDatabaseUrl, + testMemoryConfig, + type TestDb, +} from "./helpers.ts"; + +const DOCS = [ + { title: "Staging deploys", text: "Staging deploys run from main after every merge." }, + { title: "Vacation policy", text: "Request vacation two weeks ahead." }, +]; + +/** Every row of every memory table, keyed by table. */ +type Snapshot = Record; + +describe.skipIf(testDatabaseUrl() === undefined)("upgrading a 0.1.0 database", () => { + let db: TestDb; + let before: Snapshot; + const grantStore = createInMemoryGrantStore([ + allow("alice", "add"), + allow("alice", "search"), + ]); + + async function snapshot(): Promise { + const tables = await db.sql<{ name: string }[]>` + SELECT table_name AS name FROM information_schema.tables + WHERE table_schema = 'memory' AND table_type = 'BASE TABLE' + AND table_name <> '_migrations' + ORDER BY 1`; + const result: Snapshot = {}; + for (const { name } of tables) { + const rows = await db.sql<{ row: string }[]>` + SELECT to_jsonb(t)::text AS row FROM ${db.sql("memory")}.${db.sql(name)} t + ORDER BY 1`; + result[name] = rows.map((r) => r.row); + } + return result; + } + + beforeAll(async () => { + db = await createEmptyDb(); + await runMigrations(db.config, { schema: "public" }); + await v010.runMemoryMigrations(db.databaseUrl, { ftsLanguage: "english" }); + await seedPrincipal(db, "acme", "alice"); + + const legacy = v010.createMemory({ + config: testMemoryConfig(db), + grantStore, + conditionRegistry: {}, + }); + try { + for (const content of DOCS) { + await legacy.add({ tenantId: "acme", principalId: "alice", content }); + } + } finally { + await legacy.close(); + } + // A distilled claim 0.1.0 already classed as an event: the 0004 + // backfill must not reclassify it on upgrade. + await db.sql` + INSERT INTO memory.document (id, tenant_id, kind, title, adapter, external_ref) + VALUES ('doc-claim', 'acme', 'claim', 'Deploy cadence', 'distiller', 'claim-1')`; + await db.sql` + INSERT INTO memory.version + (id, tenant_id, document_id, version, content_hash, occurred_at, + created_by_kind, provenance, temporal_class) + VALUES ('ver-claim', 'acme', 'doc-claim', 1, 'claim', now(), 'agent', 'inferred', 'event')`; + await db.sql` + INSERT INTO memory.transform_config (id, tenant_id, name, version) + VALUES ('tc-1', 'acme', 'chunker', 1)`; + await db.sql` + INSERT INTO memory.transform_run (id, tenant_id, config_id, generation, status) + VALUES ('tr-1', 'acme', 'tc-1', 'gen-1', 'completed')`; + before = await snapshot(); + + const options = { schema: "public", ftsLanguage: "english" }; + await runMemoryMigrations(db.config, options); + await runMemoryMigrations(db.config, options); + }); + + afterAll(async () => { + await db?.close(); + }); + + test("keeps every row and drops the 0.1.0 migration ledger", async () => { + expect(before["document"]).toHaveLength(DOCS.length + 1); + expect(before["raw_capture"]?.length).toBeGreaterThan(0); + expect(await snapshot()).toEqual(before); + const [ledger] = await db.sql<{ name: string | null }[]>` + SELECT to_regclass('memory._migrations')::text AS name`; + expect(ledger?.name).toBeNull(); + }); + + test("finds 0.1.0 documents through this version's search", async () => { + const memory = createTestMemory(db, grantStore); + try { + const { items } = await memory.search({ + tenantId: "acme", + principalId: "alice", + query: "staging deploys", + }); + expect(items[0]?.title).toBe("Staging deploys"); + } finally { + await memory.close(); + } + }); +}); diff --git a/package.json b/package.json index f5b828b..f56ee9d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@corbits/memory", - "version": "0.1.0", + "version": "0.2.0", "description": "Mountable memory add/search/list SDK for Interchange hubs — includes resident distiller", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -61,8 +61,9 @@ "typecheck": "tsc --noEmit", "build": "tsc -p tsconfig.build.json", "prepack": "npm run build", - "test": "bun test ./src ./tests", - "test:coverage": "bun test --coverage --coverage-reporter=lcov --coverage-reporter=text ./src ./tests" + "test": "bun test ./src", + "test:e2e": "bun test ./e2e", + "test:coverage": "bun test --coverage --coverage-reporter=lcov --coverage-reporter=text ./src ./e2e" }, "dependencies": { "arktype": "^2.1.29" @@ -81,6 +82,7 @@ "postgres": "^3.4.9" }, "devDependencies": { + "@corbits/memory-0.1.0": "npm:@corbits/memory@0.1.0", "@intx/agent": "0.4.0", "@intx/authz": "0.4.0", "@intx/db": "0.4.0", diff --git a/tests/lib/http-stub.ts b/tests/lib/http-stub.ts deleted file mode 100644 index ce6ad85..0000000 --- a/tests/lib/http-stub.ts +++ /dev/null @@ -1,46 +0,0 @@ -// A local HTTP server standing in for an embed or rerank endpoint. Each test -// sets `reply`; every request is recorded with its path, auth header and -// JSON body (both clients only send JSON). - -export type StubRequest = { - path: string; - authorization: string | null; - body: unknown; -}; - -export type HttpStub = { - url: string; - requests: StubRequest[]; - reply: (req: StubRequest) => Response | Promise; - reset: () => void; - stop: () => void; -}; - -const unconfigured = () => new Response("no reply configured", { status: 500 }); - -export function startHttpStub(): HttpStub { - const stub: HttpStub = { - url: "", - requests: [], - reply: unconfigured, - reset: () => { - stub.requests.length = 0; - stub.reply = unconfigured; - }, - stop: () => server.stop(true), - }; - const server = Bun.serve({ - port: 0, - async fetch(req) { - const recorded: StubRequest = { - path: new URL(req.url).pathname, - authorization: req.headers.get("authorization"), - body: await req.json(), - }; - stub.requests.push(recorded); - return stub.reply(recorded); - }, - }); - stub.url = server.url.href.replace(/\/$/, ""); - return stub; -} diff --git a/tsconfig.json b/tsconfig.json index 94f57a4..5aac854 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,5 +16,5 @@ "forceConsistentCasingInFileNames": true, "types": ["bun"] }, - "include": ["src", "scripts", "tests"] + "include": ["src", "scripts", "e2e"] }