From 50c0a9cb980264d866d68c1816b17e7e52f331f1 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 12 Jul 2026 12:01:08 +1000 Subject: [PATCH 1/8] =?UTF-8?q?feat(stack):=20add=20EQL=20v3=20JSON=20colu?= =?UTF-8?q?mns=20(types.Json,=20ste=5Fvec)=20=E2=80=94=20model=20+=20round?= =?UTF-8?q?-trip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR4 (foundation). `types.Json('col')` declares a `public.eql_v3_json` column that encrypts a JSON document to an ste_vec `SteVecDocument` and round-trips it losslessly through encrypt/decrypt and the model path. - columns.ts: add `'json'` to the cast_as kinds, a `JsonValue` plaintext type, an `eql_v3_json` domain + `EncryptedJsonColumn` class, and a new OPTIONAL `searchableJson` query capability that emits the `ste_vec` index (the scalar 3-flag model can't express it). `isQueryable`/`QueryableFlag` count it. - The ste_vec index uses `mode: 'compat'`, which eql-3.0.0's `eql_v3_json` REQUIRES: it orders entries by the CLLW-OPE `op` term, so v2's `'standard'`/ CLLW-`oc` terms are rejected at encrypt time. (Found by the round-trip test — the FFI error names this exactly.) - types.ts: `types.Json` factory (auto-registers in the domain-registry). - test-kit: a deferred `eql_v3_json` catalog row (ste_vec containment doesn't fit the scalar op oracle — covered by a dedicated suite), a `json` family for coverage accounting, `searchableJson` in the op map, and the sample type widened to include `JsonValue`. - Updated the domain count/list assertions the 40th domain forces. Verified against live crypto: the json round-trip suite passes (2/2). Unit + type suites green (1676 + 93). Containment querying through the adapters is the next increment. Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .changeset/eql-v3-json.md | 11 +++ .../__tests__/eql-v3-domain-registry.test.ts | 1 + .../__tests__/supabase-v3-matrix.test.ts | 9 ++- .../stack/__tests__/test-kit-families.test.ts | 9 ++- .../shared/json-crypto.integration.test.ts | 46 ++++++++++++ packages/stack/src/eql/v3/columns.ts | 71 ++++++++++++++++++- packages/stack/src/eql/v3/index.ts | 2 + packages/stack/src/eql/v3/types.ts | 4 ++ packages/test-kit/src/catalog.ts | 31 +++++++- packages/test-kit/src/families.ts | 6 ++ packages/test-kit/src/ops.ts | 5 ++ packages/test-kit/src/oracle.ts | 5 +- 12 files changed, 190 insertions(+), 10 deletions(-) create mode 100644 .changeset/eql-v3-json.md create mode 100644 packages/stack/integration/shared/json-crypto.integration.test.ts diff --git a/.changeset/eql-v3-json.md b/.changeset/eql-v3-json.md new file mode 100644 index 000000000..e71b0eb6b --- /dev/null +++ b/.changeset/eql-v3-json.md @@ -0,0 +1,11 @@ +--- +'@cipherstash/stack': minor +--- + +Add EQL v3 JSON columns. `types.Json('col')` declares a `public.eql_v3_json` +column that encrypts a JSON document to an ste_vec `SteVecDocument` and +round-trips it losslessly through `encrypt`/`decrypt` and the model path. A new +`searchableJson` query capability emits the ste_vec index; the index uses +`mode: 'compat'`, which eql-3.0.0's `eql_v3_json` requires (it orders ste_vec +entries by the CLLW-OPE `op` term, so v2's `'standard'`/CLLW-`oc` terms are +rejected). Containment querying through the adapters follows. diff --git a/packages/stack/__tests__/eql-v3-domain-registry.test.ts b/packages/stack/__tests__/eql-v3-domain-registry.test.ts index 192b034e8..22665d21c 100644 --- a/packages/stack/__tests__/eql-v3-domain-registry.test.ts +++ b/packages/stack/__tests__/eql-v3-domain-registry.test.ts @@ -61,6 +61,7 @@ const EXPECTED_DOMAIN_KEYS = [ 'eql_v3_double_eq', 'eql_v3_double_ord_ore', 'eql_v3_double_ord', + 'eql_v3_json', ] as const describe('DOMAIN_REGISTRY', () => { diff --git a/packages/stack/__tests__/supabase-v3-matrix.test.ts b/packages/stack/__tests__/supabase-v3-matrix.test.ts index 876fe4ee3..3ecdb3d99 100644 --- a/packages/stack/__tests__/supabase-v3-matrix.test.ts +++ b/packages/stack/__tests__/supabase-v3-matrix.test.ts @@ -101,12 +101,15 @@ function firstSample(spec: DomainSpec): unknown { describe('supabase v3 wire encoding, every domain', () => { // Guards the tier arithmetic itself. A domain silently dropping out of a // tier would otherwise just shrink an `it.each` with no test turning red. - it('tiers all 39 domains', () => { - expect(matrixEntries).toHaveLength(39) + it('tiers all 40 domains', () => { + expect(matrixEntries).toHaveLength(40) expect(equalityDomains).toHaveLength(28) expect(orderDomains).toHaveLength(19) expect(matchDomains).toHaveLength(2) - expect(storageOnlyDomains).toHaveLength(10) + // +1: `eql_v3_json` carries only an `ste_vec` index, so from this scalar + // tiering it reads as storage-only — the Supabase adapter has no JSON + // containment path (PostgREST can't cast/call), so it rejects scalar ops. + expect(storageOnlyDomains).toHaveLength(11) }) describe.each(matrixEntries)('%s', (eqlType, spec) => { diff --git a/packages/stack/__tests__/test-kit-families.test.ts b/packages/stack/__tests__/test-kit-families.test.ts index 6a9d951c1..732a2e34e 100644 --- a/packages/stack/__tests__/test-kit-families.test.ts +++ b/packages/stack/__tests__/test-kit-families.test.ts @@ -55,7 +55,7 @@ describe('test-kit families partition the v3 catalog', () => { expect([...covered, ...deferred].sort()).toEqual([...allBare].sort()) }) - it('defers exactly the block-ORE domains, each with a reason', () => { + it('defers the block-ORE domains and json, each with a reason', () => { const deferred = FAMILY_NAMES.flatMap((f) => deferredForFamily(f)) expect(deferred.map((d) => d.bare).sort()).toEqual([ @@ -63,13 +63,18 @@ describe('test-kit families partition the v3 catalog', () => { 'date_ord_ore', 'double_ord_ore', 'integer_ord_ore', + 'json', 'numeric_ord_ore', 'real_ord_ore', 'smallint_ord_ore', 'text_ord_ore', 'timestamp_ord_ore', ]) - for (const { reason } of deferred) expect(reason).toMatch(/superuser-only/) + // ORE domains defer for the superuser-only opclass; json defers because + // ste_vec containment isn't in the scalar op matrix. + for (const { bare, reason } of deferred) { + expect(reason).toMatch(bare === 'json' ? /ste_vec/ : /superuser-only/) + } }) it('never hands a family a domain from a neighbouring prefix', () => { diff --git a/packages/stack/integration/shared/json-crypto.integration.test.ts b/packages/stack/integration/shared/json-crypto.integration.test.ts new file mode 100644 index 000000000..05b40b109 --- /dev/null +++ b/packages/stack/integration/shared/json-crypto.integration.test.ts @@ -0,0 +1,46 @@ +/** + * Live round-trip for the v3 `types.Json()` column — an encrypted JSONB document + * stored as an ste_vec `SteVecDocument`. Proves the new json column model + * encrypts and decrypts a real document through protect-ffi (no DB query here; + * containment is exercised by the Drizzle json suite). + */ +import { unwrapResult } from '@cipherstash/test-kit' +import { beforeAll, describe, expect, it } from 'vitest' +import { EncryptionV3, encryptedTable, types } from '@/encryption/v3' + +const docs = encryptedTable('v3_json_docs', { + profile: types.Json('profile'), +}) + +describe('v3 typed client — JSON (ste_vec) round-trip', () => { + let client: Awaited>> + + beforeAll(async () => { + client = await EncryptionV3({ schemas: [docs] }) + }, 30000) + + it('round-trips a JSON document through encrypt/decrypt', async () => { + const value = { + user: 'ada@example.com', + roles: ['admin', 'eng'], + active: true, + meta: { since: 2020 }, + } + + const encrypted = unwrapResult( + await client.encrypt(value, { table: docs, column: docs.profile }), + ) + // An ste_vec document carries an `sv` array, not a scalar `c` ciphertext. + expect(Array.isArray((encrypted as { sv?: unknown }).sv)).toBe(true) + + const decrypted = unwrapResult(await client.decrypt(encrypted)) + expect(decrypted).toEqual(value) + }, 30000) + + it('round-trips a JSON document through the model path', async () => { + const model = { profile: { user: 'grace@example.com', roles: ['eng'] } } + const encrypted = unwrapResult(await client.encryptModel(model, docs)) + const decrypted = unwrapResult(await client.decryptModel(encrypted, docs)) + expect(decrypted).toEqual(model) + }, 30000) +}) diff --git a/packages/stack/src/eql/v3/columns.ts b/packages/stack/src/eql/v3/columns.ts index 823bc5d2e..5c070effa 100644 --- a/packages/stack/src/eql/v3/columns.ts +++ b/packages/stack/src/eql/v3/columns.ts @@ -10,11 +10,15 @@ import { defaultMatchOpts } from '@/schema/match-defaults' * - `equality`: exact-match lookups (EQL `hm`, or comparison via `ob`). * - `orderAndRange`: comparison / range lookups (EQL `op`, or `ob` on `_ord_ore` domains). * - `freeTextSearch`: tokenised substring match (EQL `bf`). + * - `searchableJson`: encrypted-JSONB containment / selector lookups (EQL + * `ste_vec`). Optional and mutually exclusive with the scalar flags — a + * `public.eql_v3_json` document is queried by structure, not by scalar term. */ export type QueryCapabilities = Readonly<{ equality: boolean orderAndRange: boolean freeTextSearch: boolean + searchableJson?: boolean }> /** @@ -35,7 +39,13 @@ export type DateLikeCast = (typeof DATE_LIKE_CASTS)[number] /** The plaintext (TypeScript) kind a v3 domain decrypts to. A subset of the * SDK `CastAs` enum, restricted to the scalar kinds v3 domains actually use. */ -type PlaintextKind = 'string' | 'number' | 'bigint' | 'boolean' | DateLikeCast +type PlaintextKind = + | 'string' + | 'number' + | 'bigint' + | 'boolean' + | 'json' + | DateLikeCast /** * The full, literal definition of a v3 domain. This is the LOAD-BEARING type: @@ -57,6 +67,7 @@ type QueryableFlag = D['capabilities'] extends { equality: false orderAndRange: false freeTextSearch: false + searchableJson?: false | undefined } ? false : true @@ -357,6 +368,23 @@ function indexesForCapabilities( indexes.match = defaultMatchOpts() } + if (capabilities.searchableJson) { + // Encrypted-JSONB (ste_vec) index. `prefix: 'enabled'` is a sentinel the + // table's `build()` rewrites to `${tableName}/${columnName}` — the same + // scheme v2 `searchableJson` uses — so each document's selectors are scoped + // to their column. `array_index_mode: 'all'` indexes every array element. + // + // `mode: 'compat'` is REQUIRED for eql-3.0.0: `public.eql_v3_json` orders + // ste_vec entries by the CLLW-OPE `op` term, so the index must emit `op` + // (compat) terms. `'standard'` emits v2's CLLW-ORE `oc` terms, which the v3 + // domain rejects at encrypt time. + indexes.ste_vec = { + prefix: 'enabled', + array_index_mode: 'all', + mode: 'compat', + } + } + return indexes } @@ -374,7 +402,8 @@ function isQueryableCapabilities(capabilities: QueryCapabilities): boolean { return ( capabilities.equality || capabilities.orderAndRange || - capabilities.freeTextSearch + capabilities.freeTextSearch || + (capabilities.searchableJson ?? false) ) } @@ -584,6 +613,30 @@ export class EncryptedDoubleOrdColumn extends EncryptedV3Column< typeof DOUBLE_ORD > {} +// json +const JSON_DOMAIN = { + eqlType: 'public.eql_v3_json', + castAs: 'json', + capabilities: { + equality: false, + orderAndRange: false, + freeTextSearch: false, + searchableJson: true, + }, +} as const + +/** + * Builder for a `public.eql_v3_json` column — an encrypted JSONB document, + * stored as an ste_vec `SteVecDocument` and queried by containment. The document + * (any {@link JsonValue}) round-trips through `encrypt`/`decrypt`; structural + * queries use the ste_vec index emitted by `build()`. + */ +export class EncryptedJsonColumn extends EncryptedV3Column { + constructor(columnName: string) { + super(columnName, JSON_DOMAIN) + } +} + /** * Union of every v3 concrete column type. Used as the value type for v3 table * columns so a table may mix any generated domains. @@ -628,6 +681,7 @@ export type AnyEncryptedV3Column = | EncryptedDoubleEqColumn | EncryptedDoubleOrdOreColumn | EncryptedDoubleOrdColumn + | EncryptedJsonColumn /** * A factory that builds a concrete v3 column for a given DB column name. @@ -646,6 +700,15 @@ export type EncryptedV3TableColumn = { [key: string]: AnyEncryptedV3Column } +/** A JSON value — the plaintext a `public.eql_v3_json` column reconstructs to. */ +export type JsonValue = + | string + | number + | boolean + | null + | JsonValue[] + | { [key: string]: JsonValue } + /** Map a domain's {@link PlaintextKind} to its TypeScript plaintext type. */ type PlaintextFromKind = K extends 'string' ? string @@ -657,7 +720,9 @@ type PlaintextFromKind = K extends 'string' ? boolean : K extends DateLikeCast ? Date - : never + : K extends 'json' + ? JsonValue + : never /** * The plaintext type for a single v3 column, read from the literal domain diff --git a/packages/stack/src/eql/v3/index.ts b/packages/stack/src/eql/v3/index.ts index 069dc7ad4..f02656b15 100644 --- a/packages/stack/src/eql/v3/index.ts +++ b/packages/stack/src/eql/v3/index.ts @@ -12,6 +12,7 @@ export type { AnyEncryptedV3Column, EncryptedV3TableColumn, EqlTypeForColumn, + JsonValue, PlaintextForColumn, QueryCapabilities, QueryTypesForColumn, @@ -35,6 +36,7 @@ export { EncryptedIntegerEqColumn, EncryptedIntegerOrdColumn, EncryptedIntegerOrdOreColumn, + EncryptedJsonColumn, EncryptedNumericColumn, EncryptedNumericEqColumn, EncryptedNumericOrdColumn, diff --git a/packages/stack/src/eql/v3/types.ts b/packages/stack/src/eql/v3/types.ts index 43a774c38..f14def386 100644 --- a/packages/stack/src/eql/v3/types.ts +++ b/packages/stack/src/eql/v3/types.ts @@ -29,6 +29,7 @@ import { EncryptedIntegerEqColumn, EncryptedIntegerOrdColumn, EncryptedIntegerOrdOreColumn, + EncryptedJsonColumn, EncryptedNumericColumn, EncryptedNumericEqColumn, EncryptedNumericOrdColumn, @@ -184,6 +185,9 @@ export const types = { DoubleOrdOre: (name: string) => new EncryptedDoubleOrdOreColumn(name, DOUBLE_ORD_ORE), DoubleOrd: (name: string) => new EncryptedDoubleOrdColumn(name, DOUBLE_ORD), + + // json (encrypted JSONB document, ste_vec containment) + Json: (name: string) => new EncryptedJsonColumn(name), // `satisfies` is load-bearing, not decoration: `DOMAIN_REGISTRY` derives itself // by calling every value here at module load. A non-factory export would throw // during module evaluation and take the supabase introspect/schema-build/verify diff --git a/packages/test-kit/src/catalog.ts b/packages/test-kit/src/catalog.ts index 93f085fd0..46d376b52 100644 --- a/packages/test-kit/src/catalog.ts +++ b/packages/test-kit/src/catalog.ts @@ -20,6 +20,7 @@ import type { AnyEncryptedV3Column, EqlTypeForColumn, + JsonValue, QueryCapabilities, } from '@cipherstash/stack/eql/v3' import { @@ -40,6 +41,7 @@ import { EncryptedIntegerEqColumn, EncryptedIntegerOrdColumn, EncryptedIntegerOrdOreColumn, + EncryptedJsonColumn, EncryptedNumericColumn, EncryptedNumericEqColumn, EncryptedNumericOrdColumn, @@ -103,7 +105,7 @@ export type DomainSpec = Readonly<{ * tell `integer` from `double`, and a fractional value on an int-named domain is * untested territory (it would truncate against a real narrow PG column). */ - samples: ReadonlyArray + samples: ReadonlyArray /** * Values that MUST fail encryption. Number domains reject `NaN`/`±Infinity` * and `bigint` domains reject values outside the signed 64-bit (`int8`) range @@ -665,4 +667,31 @@ export const V3_MATRIX = { samples: DOUBLE_S, errorSamples: NUM_ERR, }, + + // Encrypted JSONB document (ste_vec). DEFERRED from the scalar family driver: + // it is queried by containment/selector, not the eq/ord/match ops the oracle + // models, so it is covered by a dedicated json integration suite. The row + // still pins the built shape (`cast_as: 'json'` + the ste_vec index) and + // carries representative document samples. + 'public.eql_v3_json': { + builder: types.Json, + ColumnClass: EncryptedJsonColumn, + castAs: 'json', + capabilities: { + equality: false, + orderAndRange: false, + freeTextSearch: false, + searchableJson: true, + }, + indexes: { + ste_vec: { prefix: 'enabled', array_index_mode: 'all', mode: 'compat' }, + }, + samples: [ + { user: 'ada@example.com', roles: ['admin', 'eng'], active: true }, + { user: 'grace@example.com', roles: ['eng'] }, + ], + deferred: + 'json is queried by ste_vec containment/selector, not the scalar op ' + + 'matrix — covered by a dedicated json integration suite', + }, } as const satisfies Record diff --git a/packages/test-kit/src/families.ts b/packages/test-kit/src/families.ts index e895c9521..b0fa53c10 100644 --- a/packages/test-kit/src/families.ts +++ b/packages/test-kit/src/families.ts @@ -26,6 +26,7 @@ export type FamilyName = | 'timestamp' | 'text' | 'boolean' + | 'json' /** * The bare-domain prefixes each family owns. `real-double` pairs two prefixes @@ -46,6 +47,11 @@ const FAMILY_PREFIXES: Readonly> = { timestamp: ['timestamp'], text: ['text'], boolean: ['boolean'], + // The json family's only domain (`eql_v3_json`) is `deferred`: ste_vec + // containment doesn't fit the scalar oracle, so it's covered by a dedicated + // suite, not `runFamilySuite`. The prefix still owns it for coverage + // accounting. + json: ['json'], } export const FAMILY_NAMES = Object.keys(FAMILY_PREFIXES) as FamilyName[] diff --git a/packages/test-kit/src/ops.ts b/packages/test-kit/src/ops.ts index 9bf6724d1..a6d86fb85 100644 --- a/packages/test-kit/src/ops.ts +++ b/packages/test-kit/src/ops.ts @@ -57,6 +57,11 @@ const OPS_BY_CAPABILITY = { equality: ['eq', 'ne', 'in', 'notIn'], orderAndRange: ['gt', 'gte', 'lt', 'lte', 'between', 'notBetween', 'order'], freeTextSearch: ['contains'], + // Encrypted-JSONB containment surfaces as `contains` too (doc @> subset), but + // json domains are `deferred` from this driver (ste_vec doesn't fit the scalar + // oracle) and covered by a dedicated suite, so this entry exists to satisfy + // the capability-keyed type rather than to gate a run. + searchableJson: ['contains'], } as const satisfies Record /** diff --git a/packages/test-kit/src/oracle.ts b/packages/test-kit/src/oracle.ts index 26fefaf66..3006c702f 100644 --- a/packages/test-kit/src/oracle.ts +++ b/packages/test-kit/src/oracle.ts @@ -25,7 +25,10 @@ export function plainValue( if (sample === undefined) { throw new Error(`Domain has no samples for row ${rowKey}`) } - return sample + // Samples widened to include `JsonValue` for the (deferred) json domain; the + // scalar oracle is only ever called on covered scalar domains, so this narrows + // safely. + return sample as Plain } export function comparePlain(left: Plain, right: Plain): number { From d84a5738a1f0a3e3b76489febb2b72b0f4749cec Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 12 Jul 2026 12:34:42 +1000 Subject: [PATCH 2/8] feat(stack): EQL v3 JSON containment via Drizzle @> operator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ops.contains(col, subObject)` on a `types.Json` column now answers encrypted-JSONB containment. json has no `eql_v3.contains` overload, so the operator emits `@>` with a `query_jsonb` needle built by `encryptQuery` (no ciphertext), cast to `eql_v3.query_jsonb`. The ste_vec index drops positional array terms (`array_index_mode` keeps `item`+`wildcard`, not `position`) so containment is a true jsonb subset test: `{ roles: ['x'] }` matches any document whose `roles` array contains `x` regardless of index — verified live. - src: sql-dialect `containsJson` (@> operator), operators `contains` ste_vec branch + `encryptQuery` client capability, columns.ts index - test: json-contains live suite (4), operators unit coverage (3) - test-kit: catalog json row array-index-mode Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .changeset/eql-v3-json.md | 9 +- .../__tests__/drizzle-v3/operators.test.ts | 66 ++++++++++- .../json-contains.integration.test.ts | 112 ++++++++++++++++++ packages/stack/src/eql/v3/columns.ts | 11 +- .../stack/src/eql/v3/drizzle/operators.ts | 69 ++++++++++- .../stack/src/eql/v3/drizzle/sql-dialect.ts | 13 ++ packages/test-kit/src/catalog.ts | 6 +- 7 files changed, 276 insertions(+), 10 deletions(-) create mode 100644 packages/stack/integration/drizzle-v3/json-contains.integration.test.ts diff --git a/.changeset/eql-v3-json.md b/.changeset/eql-v3-json.md index e71b0eb6b..6544cc04c 100644 --- a/.changeset/eql-v3-json.md +++ b/.changeset/eql-v3-json.md @@ -8,4 +8,11 @@ round-trips it losslessly through `encrypt`/`decrypt` and the model path. A new `searchableJson` query capability emits the ste_vec index; the index uses `mode: 'compat'`, which eql-3.0.0's `eql_v3_json` requires (it orders ste_vec entries by the CLLW-OPE `op` term, so v2's `'standard'`/CLLW-`oc` terms are -rejected). Containment querying through the adapters follows. +rejected). + +The Drizzle integration's `contains(col, subObject)` now answers encrypted-JSONB +containment on a `types.Json` column, emitting the `@>` operator with a +`query_jsonb` needle (from `encryptQuery`). The ste_vec index indexes array +elements by identity but not position, so containment is a true subset test +(`{ roles: ['x'] }` matches any document whose `roles` array contains `x`, +regardless of index). diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index 0fe344fa1..0a5d000f0 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -50,13 +50,21 @@ function setup( }), ) { const encrypt = vi.fn((...args: unknown[]) => chainable(encryptImpl(...args))) + // `encryptQuery` backs JSON containment: it returns a narrowed `query_jsonb` + // term (no ciphertext). The double returns a recognisable ste_vec-shaped term + // so a rendered `@>` query can be asserted. + const encryptQuery = vi.fn(() => + chainable( + Promise.resolve({ data: { sv: [{ s: 'sel', hm: 'h' }] } }) as never, + ), + ) // The factory's `client` parameter is the structural `{ encrypt }` surface, // so this hand-rolled double satisfies it with no cast (M1). - const client = { encrypt } + const client = { encrypt, encryptQuery } const ops = createEncryptionOperatorsV3(client, { lockContext, audit }) const dialect = new PgDialect() const render = (s: unknown) => dialect.sqlToQuery(s as SQL) - return { ops, encrypt, render } + return { ops, encrypt, encryptQuery, render } } type BulkPayload = Array<{ id?: string; plaintext: unknown }> @@ -114,6 +122,13 @@ const storageDomains = matrixEntries.filter( !spec.indexes.ope && !spec.indexes.match, ) +// Storage-only domains that also lack an ste_vec index: these reject `contains` +// outright. A json (`ste_vec`) column is storage-only for eq/order but DOES +// answer `contains` via `@>`, so it is excluded from the containment-rejection +// group and covered by its own positive assertion below. +const noContainmentDomains = storageDomains.filter( + ([, spec]) => !spec.indexes.ste_vec, +) const users = pgTable('users', { id: integer().primaryKey(), @@ -393,6 +408,51 @@ describe('createEncryptionOperatorsV3 - free-text match', () => { }) }) +describe('createEncryptionOperatorsV3 - JSON containment', () => { + const JSON_TYPE = 'public.eql_v3_json' + + // json has no `eql_v3.contains` overload: containment is the `@>` operator, + // and the needle is a narrowed `query_jsonb` term from `encryptQuery` (no + // ciphertext), cast to `eql_v3.query_jsonb`. + it('contains emits the @> operator with a query_jsonb needle', async () => { + const { ops, encryptQuery, render } = setup() + const q = render( + await ops.contains(matrixColumn(JSON_TYPE), { roles: ['eng'] }), + ) + + expect(q.sql.toLowerCase()).toContain( + `"matrix_users"."${slug(JSON_TYPE)}" operator(public.@>) $1::eql_v3.query_jsonb`, + ) + expect(q.sql).not.toContain('eql_v3.contains') + // The needle is the encryptQuery result, not a full storage envelope. + expect(q.params).toEqual([JSON.stringify({ sv: [{ s: 'sel', hm: 'h' }] })]) + expect(encryptQuery).toHaveBeenCalledTimes(1) + expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({ + queryType: 'searchableJson', + }) + }) + + it('contains surfaces an encryptQuery failure as an EncryptionOperatorError', async () => { + const { ops, encryptQuery } = setup() + encryptQuery.mockReturnValueOnce( + chainable( + Promise.resolve({ failure: { message: 'boom' } }) as never, + ) as never, + ) + await expect( + ops.contains(matrixColumn(JSON_TYPE), { roles: ['eng'] }), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + }) + + it('contains rejects a null operand before calling encryptQuery', async () => { + const { ops, encryptQuery } = setup() + await expect( + ops.contains(matrixColumn(JSON_TYPE), null), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + expect(encryptQuery).not.toHaveBeenCalled() + }) +}) + describe('createEncryptionOperatorsV3 - storage-only domains', () => { it.each(storageDomains)('%s eq throws', async (eqlType, spec) => { const { ops } = setup() @@ -401,7 +461,7 @@ describe('createEncryptionOperatorsV3 - storage-only domains', () => { ).rejects.toBeInstanceOf(EncryptionOperatorError) }) - it.each(storageDomains)('%s contains throws', async (eqlType, spec) => { + it.each(noContainmentDomains)('%s contains throws', async (eqlType, spec) => { const { ops } = setup() await expect( ops.contains(matrixColumn(eqlType), sampleFor(spec)), diff --git a/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts b/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts new file mode 100644 index 000000000..f0dae84c3 --- /dev/null +++ b/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts @@ -0,0 +1,112 @@ +/** + * Live JSON containment for the v3 `types.Json()` column through the Drizzle + * operators. Seeds encrypted JSONB documents and queries them with + * `ops.contains(col, subObject)` — the same operator text search uses, dispatched + * to `eql_v3.contains` over the ste_vec document — asserting it returns exactly + * the rows whose document contains the sub-object (jsonb `@>` semantics), and + * excludes the rest. + */ +import { databaseUrl, unwrapResult } from '@cipherstash/test-kit' +import { and, asc as drizzleAsc, eq as drizzleEq, type SQL } from 'drizzle-orm' +import { integer, pgTable, text } from 'drizzle-orm/pg-core' +import { drizzle } from 'drizzle-orm/postgres-js' +import postgres from 'postgres' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { EncryptionV3, types } from '@/encryption/v3' +import { + createEncryptionOperatorsV3, + extractEncryptionSchemaV3, +} from '@/eql/v3/drizzle' +import { makeEqlV3Column } from '@/eql/v3/drizzle/column' + +const sqlClient = postgres(databaseUrl(), { prepare: false }) + +const TABLE_NAME = 'protect_ci_v3_json_contains' +const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + +const docTable = pgTable(TABLE_NAME, { + id: integer('id').primaryKey().generatedAlwaysAsIdentity(), + rowKey: text('row_key').notNull(), + testRunId: text('test_run_id').notNull(), + doc: makeEqlV3Column(types.Json('doc')), +} as never) + +const schema = extractEncryptionSchemaV3(docTable) + +// Distinct documents so each containment query has a definite expected set. +const DOCS: Record> = { + ada: { user: 'ada@example.com', roles: ['admin', 'eng'], active: true }, + grace: { user: 'grace@example.com', roles: ['eng'] }, + zoe: { user: 'zoe@example.com', roles: ['ops'], active: false }, +} + +type SelectRow = { rowKey: string } +let client: Awaited> +let ops: ReturnType +let db: ReturnType + +async function matching(condition: SQL): Promise { + const rows = (await db + .select({ rowKey: docTable.rowKey }) + .from(docTable) + .where(and(drizzleEq(docTable.testRunId, RUN), condition)) + .orderBy(drizzleAsc(docTable.rowKey))) as SelectRow[] + return rows.map((row) => row.rowKey) +} + +beforeAll(async () => { + // EQL v3 is installed once per run by `global-setup.ts`. + client = await EncryptionV3({ schemas: [schema] }) + ops = createEncryptionOperatorsV3(client) + db = drizzle({ client: sqlClient }) + + await sqlClient.unsafe(`DROP TABLE IF EXISTS ${TABLE_NAME}`) + await sqlClient.unsafe(` + CREATE TABLE ${TABLE_NAME} ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + row_key TEXT NOT NULL, + test_run_id TEXT NOT NULL, + doc public.eql_v3_json NOT NULL + ) + `) + + const rows = Object.entries(DOCS).map(([rowKey, doc]) => ({ + rowKey, + testRunId: RUN, + doc, + })) + const encrypted = unwrapResult( + await client.bulkEncryptModels(rows as never, schema), + ) as Array> + await db.insert(docTable).values(encrypted as never) +}, 120000) + +afterAll(async () => { + await sqlClient`DELETE FROM ${sqlClient(TABLE_NAME)} WHERE test_run_id = ${RUN}` + await sqlClient.end() +}, 30000) + +describe('v3 drizzle JSON containment (live pg)', () => { + it('matches every document containing a sub-object (array element containment)', async () => { + // `{ roles: ['eng'] }` ⊆ ada (admin,eng) and grace (eng), not zoe (ops). + const condition = await ops.contains(docTable.doc, { roles: ['eng'] }) + expect(await matching(condition)).toEqual(['ada', 'grace']) + }, 30000) + + it('matches on a scalar field', async () => { + const condition = await ops.contains(docTable.doc, { + user: 'zoe@example.com', + }) + expect(await matching(condition)).toEqual(['zoe']) + }, 30000) + + it('matches on a nested boolean', async () => { + const condition = await ops.contains(docTable.doc, { active: true }) + expect(await matching(condition)).toEqual(['ada']) + }, 30000) + + it('returns nothing for a sub-object no document contains', async () => { + const condition = await ops.contains(docTable.doc, { roles: ['nope'] }) + expect(await matching(condition)).toEqual([]) + }, 30000) +}) diff --git a/packages/stack/src/eql/v3/columns.ts b/packages/stack/src/eql/v3/columns.ts index 5c070effa..527891215 100644 --- a/packages/stack/src/eql/v3/columns.ts +++ b/packages/stack/src/eql/v3/columns.ts @@ -372,7 +372,14 @@ function indexesForCapabilities( // Encrypted-JSONB (ste_vec) index. `prefix: 'enabled'` is a sentinel the // table's `build()` rewrites to `${tableName}/${columnName}` — the same // scheme v2 `searchableJson` uses — so each document's selectors are scoped - // to their column. `array_index_mode: 'all'` indexes every array element. + // to their column. + // + // `array_index_mode` indexes array elements by identity (`item`) and enables + // `$[*]` wildcard selectors, but deliberately drops `position`: `@>` + // containment must be a subset test (jsonb `@>` semantics), so `{roles:['x']}` + // matches any document whose `roles` array contains `x` REGARDLESS of index. + // With positional terms emitted (v2 `searchableJson`'s `'all'`), the needle + // for `x`-at-index-0 would miss a document holding `x` at index 1. // // `mode: 'compat'` is REQUIRED for eql-3.0.0: `public.eql_v3_json` orders // ste_vec entries by the CLLW-OPE `op` term, so the index must emit `op` @@ -380,7 +387,7 @@ function indexesForCapabilities( // domain rejects at encrypt time. indexes.ste_vec = { prefix: 'enabled', - array_index_mode: 'all', + array_index_mode: { item: true, wildcard: true, position: false }, mode: 'compat', } } diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts index 251f4e4b3..ee13c6758 100644 --- a/packages/stack/src/eql/v3/drizzle/operators.ts +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -43,6 +43,8 @@ const MAX_IN_ARRAY_CONCURRENCY = 4 * * `bulkEncrypt` is optional so a `{ encrypt }`-only client stays valid; the * list operators fall back to bounded-concurrency single encryption without it. + * `encryptQuery` is likewise optional — only JSON containment (`@>`) needs it, + * to build a `query_jsonb` needle; the `contains()` branch guards its absence. */ type OperandEncryptionClient = { encrypt( @@ -53,6 +55,14 @@ type OperandEncryptionClient = { plaintexts: never, opts: { table: AnyV3Table; column: AnyEncryptedV3Column }, ): unknown + encryptQuery?( + value: never, + opts: { + table: AnyV3Table + column: AnyEncryptedV3Column + queryType: 'searchableJson' + }, + ): unknown } /** @@ -205,7 +215,7 @@ export function createEncryptionOperatorsV3( */ function requireIndex( ctx: ColumnContext, - indexes: readonly ('unique' | 'ore' | 'ope' | 'match')[], + indexes: readonly ('unique' | 'ore' | 'ope' | 'match' | 'ste_vec')[], operator: string, capability: string, ): void { @@ -223,7 +233,12 @@ export function createEncryptionOperatorsV3( // order-capable column answers equality via its ordering term too. const EQUALITY_INDEXES = ['unique', 'ore', 'ope'] as const const ORDERING_INDEXES = ['ore', 'ope'] as const - const MATCH_INDEXES = ['match'] as const + // `contains` answers two shapes: bloom free-text (`match`, a `text_search`/ + // `text_match` column), which emits `eql_v3.contains(col, operand)`, and + // encrypted-JSONB containment (`ste_vec`, a `json` column), which emits the + // `@>` operator instead — json has no `eql_v3.contains` overload. The branch + // in `contains()` picks the right SQL by index kind. + const CONTAINMENT_INDEXES = ['match', 'ste_vec'] as const function applyOperationOptions( op: ChainableOperation, @@ -409,12 +424,60 @@ export function createEncryptionOperatorsV3( opts?: EncryptionOperatorCallOpts, ): Promise { const ctx = resolveContext(left, operator) - requireIndex(ctx, MATCH_INDEXES, operator, 'free-text search') + requireIndex( + ctx, + CONTAINMENT_INDEXES, + operator, + 'free-text search or JSON containment', + ) + + // JSON containment. `eql_v3_json` has no `eql_v3.contains` overload — its + // containment is the `@>` operator, whose `(eql_v3_json, eql_v3.query_jsonb)` + // form takes a NARROWED query term from `encryptQuery` (searchableJson → no + // ciphertext), not the full storage envelope. The dialect casts it to + // `eql_v3.query_jsonb` and emits `@>`. + if (ctx.indexes.ste_vec) { + const needle = await encryptJsonContainmentTerm(ctx, right, operator) + return v3Dialect.containsJson(colSql(left), needle) + } + + // Bloom free-text (text_search / text_match): the answerable-needle rule + // applies, and the full-envelope operand disambiguates its own overload. requireAnswerableNeedle(ctx, right, operator) const enc = await encryptOperand(ctx, right, operator, opts) return v3Dialect.contains(colSql(left), enc) } + /** + * Build a `query_jsonb` containment needle for a `json` column. Uses + * `encryptQuery` (not `encrypt`): the ste_vec query term carries no ciphertext + * and satisfies the `eql_v3.query_jsonb` CHECK the `@>` overload needs. + */ + async function encryptJsonContainmentTerm( + ctx: ColumnContext, + value: unknown, + operator: string, + ): Promise { + requireNonNullOperand(ctx, value, operator) + const encryptQuery = client.encryptQuery?.bind(client) + if (!encryptQuery) { + throw operandFailure( + ctx, + operator, + 'this client does not support encryptQuery, which JSON containment requires', + ) + } + const result = (await encryptQuery(value as never, { + table: ctx.table, + column: ctx.builder, + queryType: 'searchableJson', + })) as { failure?: { message: string }; data?: unknown } + if (result.failure) { + throw operandFailure(ctx, operator, result.failure.message) + } + return sql`${JSON.stringify(result.data)}` + } + async function inArrayOp( left: SQLWrapper, values: unknown[], diff --git a/packages/stack/src/eql/v3/drizzle/sql-dialect.ts b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts index 50e2a4b05..b88550d66 100644 --- a/packages/stack/src/eql/v3/drizzle/sql-dialect.ts +++ b/packages/stack/src/eql/v3/drizzle/sql-dialect.ts @@ -41,6 +41,19 @@ export const v3Dialect = { return sql`${fn('contains')}(${left}, ${enc}::jsonb)` }, + /** + * Encrypted-JSONB containment (`eql_v3_json @> sub-document`). Unlike text, + * json has NO `eql_v3.contains` overload — containment is the `@>` operator, + * whose `(eql_v3_json, eql_v3.query_jsonb)` form takes a `query_jsonb` needle + * (from `encryptQuery`, no ciphertext). The needle is cast to + * `eql_v3.query_jsonb` EXPLICITLY: `eql_v3_json @> ?` has four RHS overloads + * (`eql_v3_json`, `eql_v3_jsonb_entry`, `jsonb`, `query_jsonb`), so a bare + * `::jsonb` is ambiguous ("operator is not unique", 42725). + */ + containsJson(left: SQL, enc: SQL): SQL { + return sql`${left} OPERATOR(public.@>) ${enc}::eql_v3.query_jsonb` + }, + orderBy(left: SQL, flavour: 'ope' | 'ore'): SQL { // eql-3.0.0 splits the ordering extractor by term flavour: `ord_term` // takes the OPE-backed `_ord` domains (returns eql_v3_internal.ope_cllw), diff --git a/packages/test-kit/src/catalog.ts b/packages/test-kit/src/catalog.ts index 46d376b52..0ee9c13bd 100644 --- a/packages/test-kit/src/catalog.ts +++ b/packages/test-kit/src/catalog.ts @@ -684,7 +684,11 @@ export const V3_MATRIX = { searchableJson: true, }, indexes: { - ste_vec: { prefix: 'enabled', array_index_mode: 'all', mode: 'compat' }, + ste_vec: { + prefix: 'enabled', + array_index_mode: { item: true, wildcard: true, position: false }, + mode: 'compat', + }, }, samples: [ { user: 'ada@example.com', roles: ['admin', 'eng'], active: true }, From 82f2e69771c707b1b50c632988ea0a11e9281075 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 12 Jul 2026 12:48:13 +1000 Subject: [PATCH 3/8] docs(skills): document EQL v3 types.Json + JSON containment - stash-encryption: types.Json in the capability-suffix and family tables, plus an "Encrypted-JSONB Queries" section (containment + JSONPath selector via encryptQuery searchableJson) - stash-drizzle: contains() answers JSON containment on a Json column (@> operator, query_jsonb needle, position-independent array subset) - changeset: stash patch (skills ship in the stash tarball) Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .changeset/eql-v3-json-skills.md | 8 ++++++++ skills/stash-drizzle/SKILL.md | 3 ++- skills/stash-encryption/SKILL.md | 27 ++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .changeset/eql-v3-json-skills.md diff --git a/.changeset/eql-v3-json-skills.md b/.changeset/eql-v3-json-skills.md new file mode 100644 index 000000000..979c4e672 --- /dev/null +++ b/.changeset/eql-v3-json-skills.md @@ -0,0 +1,8 @@ +--- +'stash': patch +--- + +Document EQL v3 JSON columns in the bundled skills: `types.Json` in the +`stash-encryption` typed-schema catalog (capability suffix, family, and an +encrypted-JSONB query section), and `contains(col, subObject)` JSON containment +on the v3 Drizzle operators in `stash-drizzle`. diff --git a/skills/stash-drizzle/SKILL.md b/skills/stash-drizzle/SKILL.md index 206bf8bef..856e06894 100644 --- a/skills/stash-drizzle/SKILL.md +++ b/skills/stash-drizzle/SKILL.md @@ -774,13 +774,14 @@ const dec = await client.bulkDecryptModels(rows, usersSchema) |---|---| | `eq`, `ne`, `inArray`, `notInArray` | equality (`Eq`, `Ord`, `OrdOre`, `TextSearch`) | | `gt`, `gte`, `lt`, `lte`, `between`, `notBetween`, `asc`, `desc` | order/range (`Ord`, `OrdOre`, `TextSearch`) | -| `contains` | free-text (`TextMatch`, `TextSearch`) | +| `contains` | free-text (`TextMatch`, `TextSearch`) **or** encrypted-JSONB containment (`Json`) | | `and`, `or` | combinators — accept lazy (un-awaited) operators and `undefined`, resolve concurrently | | `isNull`, `isNotNull`, `not`, `exists`, `notExists` | Drizzle passthroughs, no encryption | Differences from the v2 operators to know about: - **`like` / `ilike` do not exist — by design.** v3 free-text search is tokenised containment, not SQL pattern matching; `contains(col, needle)` is the free-text operator. Don't pass `%` wildcards. +- **`contains` on a `types.Json` column** answers encrypted-JSONB containment instead of free-text: `contains(col, { roles: ['admin'] })` matches every row whose document contains that sub-object (jsonb `@>` semantics; array containment is position-independent). It emits the `@>` operator with a `query_jsonb` needle — a `Json` column carries no `eq` / ordering, so those operators throw on it. - **No plaintext-column fallback.** Every v3 operator requires an encrypted v3 column and throws `EncryptionOperatorError` otherwise. Use regular Drizzle operators for non-encrypted columns. - A `null` operand throws — use `isNull()` / `isNotNull()` for NULL checks. - `inArray` / `notInArray` reject an empty list, and encrypt the whole list in a single `bulkEncrypt` crossing when the client exposes one. diff --git a/skills/stash-encryption/SKILL.md b/skills/stash-encryption/SKILL.md index a2aee1353..af6d5db03 100644 --- a/skills/stash-encryption/SKILL.md +++ b/skills/stash-encryption/SKILL.md @@ -632,6 +632,7 @@ Each factory in `types` maps 1:1 to a Postgres domain named `public.eql_v3_` semantics). Array containment is a subset test regardless of + element position: `{ roles: ['admin'] }` matches any document whose `roles` + array includes `admin`. +- **JSONPath selector** — pass a `'$.path'` string to query into the document. + +```typescript +const events = encryptedTable("events", { metadata: types.Json("metadata") }) + +// containment: object needle +await client.encryptQuery({ roles: ["admin"] }, { column: events.metadata, table: events }) +``` + +Through the Drizzle v3 integration this is `ops.contains(col, subObject)` — see +the `stash-drizzle` skill. `types.Json` carries no equality or ordering, so +`eq` / `gt` / `asc` on it throw. + ### Strongly-Typed Client: `EncryptionV3` `EncryptionV3` from `@cipherstash/stack/v3` returns a `TypedEncryptionClient` whose method signatures are derived from your schemas — wrong-typed plaintext is rejected at compile time, and query methods only accept queryable columns with `queryType` constrained to the column's capabilities: From df5a532d8c1969a74373d2bcb5538418228fc631 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 12 Jul 2026 12:59:44 +1000 Subject: [PATCH 4/8] fix(stack): correct v3 JSON payload/type shapes surfaced by CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI failures, both from the JSON domain now living in the matrix: - matrix-crypto round-trip asserted every encrypted field has a top-level `c`. A JSON (`eql_v3_json`) value is an ste_vec document (`{k:'sv', sv:[...]}`) with no top-level `c` — the ciphertext is inside `sv`. Guard json by its `sv` array instead, mirroring the json-crypto suite. - The structural `OperandEncryptionClient.encryptQuery` pinned `queryType: 'searchableJson'`, which the real client's generic `encryptQuery` (queryType bound to the column's own query types) could not satisfy, so `TypedEncryptionClient` stopped assigning. Use `never` operands like `encrypt`; the call site casts. Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .../shared/matrix-crypto.integration.test.ts | 14 +++++++--- .../stack/src/eql/v3/drizzle/operators.ts | 27 ++++++++++--------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/packages/stack/integration/shared/matrix-crypto.integration.test.ts b/packages/stack/integration/shared/matrix-crypto.integration.test.ts index c457b82e7..11dcd4432 100644 --- a/packages/stack/integration/shared/matrix-crypto.integration.test.ts +++ b/packages/stack/integration/shared/matrix-crypto.integration.test.ts @@ -87,9 +87,17 @@ describe('v3 matrix live round-trip (all domains × samples)', () => { it.each( roundTripCases, )('%s round-trips through the model path', (_label, col, sample, i) => { - // Guard against a false pass: the field must be a real ciphertext (`c`), - // not a plaintext value that slipped through un-encrypted. - expect(encrypted[i][col]).toHaveProperty('c') + // Guard against a false pass: the field must be a real ciphertext, not a + // plaintext value that slipped through un-encrypted. Scalar domains carry it + // at a top-level `c`; a JSON (`eql_v3_json`) document is an ste_vec payload + // (`{ k: 'sv', sv: [...] }`) with NO top-level `c`, so it is guarded by the + // presence of its `sv` array instead. + const payload = encrypted[i][col] as { k?: unknown; sv?: unknown } + if (payload?.k === 'sv') { + expect(Array.isArray(payload.sv)).toBe(true) + } else { + expect(payload).toHaveProperty('c') + } const actual = decrypted[i][col] if (sample instanceof Date) { diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts index ee13c6758..d2e817354 100644 --- a/packages/stack/src/eql/v3/drizzle/operators.ts +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -55,14 +55,12 @@ type OperandEncryptionClient = { plaintexts: never, opts: { table: AnyV3Table; column: AnyEncryptedV3Column }, ): unknown - encryptQuery?( - value: never, - opts: { - table: AnyV3Table - column: AnyEncryptedV3Column - queryType: 'searchableJson' - }, - ): unknown + // `never` operands, like `encrypt` above: the real client's `encryptQuery` is + // generic (`queryType` is constrained to the column's own query types), which a + // concrete signature here cannot match. `never` params keep the structural + // surface satisfiable by that generic method AND by a test double. The call + // site casts its real operands. + encryptQuery?(value: never, opts: never): unknown } /** @@ -467,11 +465,14 @@ export function createEncryptionOperatorsV3( 'this client does not support encryptQuery, which JSON containment requires', ) } - const result = (await encryptQuery(value as never, { - table: ctx.table, - column: ctx.builder, - queryType: 'searchableJson', - })) as { failure?: { message: string }; data?: unknown } + const result = (await encryptQuery( + value as never, + { + table: ctx.table, + column: ctx.builder, + queryType: 'searchableJson', + } as never, + )) as { failure?: { message: string }; data?: unknown } if (result.failure) { throw operandFailure(ctx, operator, result.failure.message) } From bf526c4158585cbe84ef28ebe6a989377fab8030 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 12 Jul 2026 13:06:44 +1000 Subject: [PATCH 5/8] fix(test-kit): document fully-deferred families instead of throwing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The family-suite driver threw in planTable when a family had zero covered domains. The `json` family is exactly that by design — ste_vec containment/selector queries don't fit the scalar oracle, so its one domain is deferred and covered by dedicated suites (json-crypto, json-contains). runFamilySuite now emits a single assertion documenting the deferral for such a family (keeping it visible, not a silent skip), while a family with no domains at all still falls through to the throw as a wiring-bug guard. Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- packages/test-kit/src/run-family-suite.ts | 26 ++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/test-kit/src/run-family-suite.ts b/packages/test-kit/src/run-family-suite.ts index f220e584a..c17cb3278 100644 --- a/packages/test-kit/src/run-family-suite.ts +++ b/packages/test-kit/src/run-family-suite.ts @@ -1,7 +1,11 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest' import type { IntegrationAdapter } from './adapter.ts' import type { DomainSpec } from './catalog.ts' -import { domainsForFamily, type FamilyName } from './families.ts' +import { + deferredForFamily, + domainsForFamily, + type FamilyName, +} from './families.ts' import { negativeOps, type Plain, positiveOps, type QueryOp } from './ops.ts' import { comparePlain, @@ -147,6 +151,26 @@ export function runFamilySuite( const adapter = makeAdapter() const domains = domainsForFamily(family) + // A family whose every domain is deferred (e.g. `json`: ste_vec containment + // and selector queries don't fit the scalar oracle this driver runs). It + // carries no op-matrix coverage here — its real coverage lives in dedicated + // suites (`json-crypto`, `json-contains`). Emit one assertion documenting the + // deferral so the family stays visible and is not a silent skip, rather than + // letting `planTable` throw on an empty column set. A family with NO domains + // at all (covered or deferred) is a wiring bug and still falls through to that + // throw. + if (domains.length === 0) { + const deferred = deferredForFamily(family) + if (deferred.length > 0) { + describe(`v3 ${adapter.name} — ${family}`, () => { + it('is fully deferred from the op-matrix; covered by dedicated suites', () => { + for (const { reason } of deferred) expect(reason).not.toBe('') + }) + }) + return + } + } + // Unique per run so a crashed run never leaves a table that shadows the next. const runId = Math.random().toString(36).slice(2, 8) const table = planTable(family, domains, runId) From 05e0cf85781a49b09c0c73bf6d9801fd22d4485b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 12 Jul 2026 13:27:53 +1000 Subject: [PATCH 6/8] test(stack): update supabase introspect for the now-modelled json domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eql_v3_json is a modelled domain now (types.Json → in DOMAIN_REGISTRY), so introspection correctly stops reporting a `doc eql_v3_json` column as unmodelled. The suite predated json existing: - the UNMODELLED table's `doc` is now a modelled column (synthesized, columnBuilders=['doc']); only `score` (eql_v3_integer_ord_ope, still no factory) is reported unmodelled - the factory sanity check keyed by BARE names ('json') which never match the registry's `eql_v3_*` keys — a latent no-op. Key it by the real `eql_v3_*` domain name so it actually asserts json is modelled and integer_ord_ope is not Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .../supabase/introspect.integration.test.ts | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/stack/integration/supabase/introspect.integration.test.ts b/packages/stack/integration/supabase/introspect.integration.test.ts index 69d394d60..e49dcf4e9 100644 --- a/packages/stack/integration/supabase/introspect.integration.test.ts +++ b/packages/stack/integration/supabase/introspect.integration.test.ts @@ -28,7 +28,10 @@ beforeAll(async () => { // as unmodelled — it is an ordinary plaintext passthrough. await sql.unsafe(`DROP DOMAIN IF EXISTS public.${USER_DOMAIN}`) await sql.unsafe(`CREATE DOMAIN public.${USER_DOMAIN} AS jsonb`) - // Columns typed with EQL domains that have NO types factory. + // Mixed EQL columns: `score` is typed with an EQL domain the SDK has NO types + // factory for (unmodelled → reported); `doc` is `eql_v3_json`, which `types.Json` + // DOES model (so it must be synthesized, not reported); `own` is a user's plain + // jsonb domain (plaintext passthrough, never reported). await sql.unsafe(` CREATE TABLE ${UNMODELLED} ( id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, @@ -92,13 +95,16 @@ describe('eql_v3 supabase introspection', () => { it('reports unmodelled EQL columns, keyed by table', async () => { const { unmodelled } = await introspect(databaseUrl()) - // Sanity: these really are EQL domains with no types factory. - expect(factoryForDomain('integer_ord_ope')).toBeUndefined() - expect(factoryForDomain('json')).toBeUndefined() + // Sanity, keyed exactly as the registry keys and the introspection query do + // (the full `eql_v3_*` domain name): `eql_v3_integer_ord_ope` has no types + // factory (unmodelled → reported), whereas `eql_v3_json` now HAS one + // (`types.Json` → modelled → NOT reported). + expect(factoryForDomain('eql_v3_integer_ord_ope')).toBeUndefined() + expect(factoryForDomain('eql_v3_json')).toBeDefined() const offenders = unmodelled.get(UNMODELLED) expect(offenders).toBeDefined() - expect(offenders?.map((c) => c.columnName).sort()).toEqual(['doc', 'score']) + expect(offenders?.map((c) => c.columnName).sort()).toEqual(['score']) expect(offenders?.find((c) => c.columnName === 'score')?.domainName).toBe( 'eql_v3_integer_ord_ope', ) @@ -119,11 +125,13 @@ describe('eql_v3 supabase introspection', () => { // The precondition that makes the `from()` guard load-bearing: an unmodelled // column is silently dropped from the encrypt config, yet stays in // `allColumns` — so `select('*')` would select it and return raw ciphertext. + // `doc` (modelled json) gets a builder; `score` (unmodelled) is dropped from + // the config but retained in `allColumns`. it('synthesizeTables drops an unmodelled column but allColumns keeps it', async () => { const { tables } = await introspect(databaseUrl()) const { tables: synth, allColumns } = synthesizeTables(tables) - expect(Object.keys(synth.get(UNMODELLED)!.columnBuilders)).toEqual([]) + expect(Object.keys(synth.get(UNMODELLED)!.columnBuilders)).toEqual(['doc']) expect(allColumns.get(UNMODELLED)).toContain('score') expect(allColumns.get(UNMODELLED)).toContain('doc') }, 30000) From 58b9ef8ba80b40e3b8ffe13193ad5c999e90d077 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 12 Jul 2026 15:09:36 +1000 Subject: [PATCH 7/8] fix(stack): address v3 JSON review findings (guards, type, tests, doc) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on PR #621 (coderdan test-gaps + Copilot): - Empty-object containment leak: `contains(col, {})` matched every row (`doc @> '{}'` holds for all), the whole-table footgun the bloom path already guards. Reject an empty-object needle in the ste_vec path with a typed EncryptionOperatorError; pin it in both the unit and live suites. - Type/runtime mismatch: `types.Json` plaintext was `JsonValue` (admits top-level scalars), but protect-ffi rejects a bare scalar ("Cannot convert … to Json"). Narrow the column plaintext to a new `JsonDocument` (object | array | null); keep `JsonValue` for nested values. Export it. - encryptQuery-absent guard was untested (the double always supplied it). Add a unit test with a `{ encrypt }`-only client. - Non-object roots unpinned: add live round-trips for array and null roots, and assert the top-level-scalar rejection so the boundary is explicit. - Copilot: json-contains header said `eql_v3.contains`; it's the `@>` operator. Corrected. Skill updated to describe JsonDocument. Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .../__tests__/drizzle-v3/operators.test.ts | 24 ++++++++++++ .../json-contains.integration.test.ts | 19 +++++++-- .../shared/json-crypto.integration.test.ts | 39 +++++++++++++++++++ packages/stack/src/eql/v3/columns.ts | 14 ++++++- .../stack/src/eql/v3/drizzle/operators.ts | 16 ++++++++ packages/stack/src/eql/v3/index.ts | 1 + skills/stash-encryption/SKILL.md | 2 +- 7 files changed, 108 insertions(+), 7 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index 0a5d000f0..06b79c2af 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -451,6 +451,30 @@ describe('createEncryptionOperatorsV3 - JSON containment', () => { ).rejects.toBeInstanceOf(EncryptionOperatorError) expect(encryptQuery).not.toHaveBeenCalled() }) + + // `doc @> '{}'` holds for every row (jsonb `{} ⊆ anything`); an empty-object + // needle would silently return the whole table, so it is refused before + // encrypting — the same whole-table guard the bloom path applies. + it('contains rejects an empty-object needle before calling encryptQuery', async () => { + const { ops, encryptQuery } = setup() + await expect(ops.contains(matrixColumn(JSON_TYPE), {})).rejects.toThrow( + /matches every row/, + ) + expect(encryptQuery).not.toHaveBeenCalled() + }) + + // `encryptQuery` is OPTIONAL on the operand client (see OperandEncryptionClient): + // a `{ encrypt }`-only client is structurally valid but cannot build the + // `query_jsonb` needle JSON containment needs. Without this guard the call would + // be a raw `TypeError: encryptQuery is not a function`; the guard turns it into a + // typed EncryptionOperatorError. Exercises the branch the real double can't. + it('contains throws a typed error when the client lacks encryptQuery', async () => { + const encrypt = vi.fn(() => chainable(Promise.resolve({ data: TERM }))) + const ops = createEncryptionOperatorsV3({ encrypt }) + await expect( + ops.contains(matrixColumn(JSON_TYPE), { roles: ['eng'] }), + ).rejects.toBeInstanceOf(EncryptionOperatorError) + }) }) describe('createEncryptionOperatorsV3 - storage-only domains', () => { diff --git a/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts b/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts index f0dae84c3..ad9640176 100644 --- a/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts +++ b/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts @@ -1,10 +1,11 @@ /** * Live JSON containment for the v3 `types.Json()` column through the Drizzle * operators. Seeds encrypted JSONB documents and queries them with - * `ops.contains(col, subObject)` — the same operator text search uses, dispatched - * to `eql_v3.contains` over the ste_vec document — asserting it returns exactly - * the rows whose document contains the sub-object (jsonb `@>` semantics), and - * excludes the rest. + * `ops.contains(col, subObject)` — the same operator text search uses, but on a + * `json` column it emits the `@>` operator over the ste_vec document (json has + * no `eql_v3.contains` overload) with a `query_jsonb` needle from + * `encryptQuery` — asserting it returns exactly the rows whose document contains + * the sub-object (jsonb `@>` semantics), and excludes the rest. */ import { databaseUrl, unwrapResult } from '@cipherstash/test-kit' import { and, asc as drizzleAsc, eq as drizzleEq, type SQL } from 'drizzle-orm' @@ -109,4 +110,14 @@ describe('v3 drizzle JSON containment (live pg)', () => { const condition = await ops.contains(docTable.doc, { roles: ['nope'] }) expect(await matching(condition)).toEqual([]) }, 30000) + + // `doc @> '{}'` holds for every row (jsonb `{} ⊆ anything`), so an empty-object + // needle would silently return the whole table — the same whole-table footgun + // the bloom path rejects. The operator must refuse it before querying, rather + // than leak every row. + it('rejects an empty-object needle before querying', async () => { + await expect(ops.contains(docTable.doc, {})).rejects.toThrow( + /matches every row/, + ) + }, 30000) }) diff --git a/packages/stack/integration/shared/json-crypto.integration.test.ts b/packages/stack/integration/shared/json-crypto.integration.test.ts index 05b40b109..f240d8754 100644 --- a/packages/stack/integration/shared/json-crypto.integration.test.ts +++ b/packages/stack/integration/shared/json-crypto.integration.test.ts @@ -43,4 +43,43 @@ describe('v3 typed client — JSON (ste_vec) round-trip', () => { const decrypted = unwrapResult(await client.decryptModel(encrypted, docs)) expect(decrypted).toEqual(model) }, 30000) + + // A JSON document is not only an object: `types.Json` (and the `JsonDocument` + // plaintext type) also admit a top-level array and `null`. Pin both so a + // regression in ste_vec encoding of non-object roots turns a test red. + it.each([ + ['array root', [1, 'two', { three: 3 }]], + ['null root', null], + ] as const)( + 'round-trips a %s', + async (_label, value) => { + const encrypted = unwrapResult( + await client.encrypt(value, { table: docs, column: docs.profile }), + ) + const decrypted = unwrapResult(await client.decrypt(encrypted)) + expect(decrypted).toEqual(value) + }, + 30000, + ) + + // The boundary the `JsonDocument` type encodes: a top-level SCALAR is not a + // JSON document. protect-ffi rejects it ("Cannot convert … to Json") — a bare + // scalar belongs in a scalar domain (`types.TextEq`, `types.IntegerEq`, …). + // `as never` bypasses the compile-time block to prove the runtime enforces it. + it.each([ + ['string', 'hello'], + ['number', 42], + ['boolean', true], + ] as const)( + 'rejects a top-level %s scalar at encrypt', + async (_label, scalar) => { + const result = await client.encrypt(scalar as never, { + table: docs, + column: docs.profile, + }) + expect(result.failure).toBeDefined() + expect(result.failure?.message).toMatch(/convert .* to Json/i) + }, + 30000, + ) }) diff --git a/packages/stack/src/eql/v3/columns.ts b/packages/stack/src/eql/v3/columns.ts index 527891215..2e53e1ca0 100644 --- a/packages/stack/src/eql/v3/columns.ts +++ b/packages/stack/src/eql/v3/columns.ts @@ -707,7 +707,7 @@ export type EncryptedV3TableColumn = { [key: string]: AnyEncryptedV3Column } -/** A JSON value — the plaintext a `public.eql_v3_json` column reconstructs to. */ +/** A JSON value at any depth. Used for the values NESTED inside a document. */ export type JsonValue = | string | number @@ -716,6 +716,16 @@ export type JsonValue = | JsonValue[] | { [key: string]: JsonValue } +/** + * The plaintext a `public.eql_v3_json` column encrypts and reconstructs: a JSON + * DOCUMENT — an object, an array, or `null`. NOT a bare top-level scalar: + * protect-ffi's `json` cast rejects a top-level string/number/boolean + * ("Cannot convert … to Json"), because a scalar belongs in a scalar domain + * (`types.TextEq`, `types.IntegerEq`, …). Nested scalars are ordinary + * {@link JsonValue}s and are fully supported. + */ +export type JsonDocument = null | JsonValue[] | { [key: string]: JsonValue } + /** Map a domain's {@link PlaintextKind} to its TypeScript plaintext type. */ type PlaintextFromKind = K extends 'string' ? string @@ -728,7 +738,7 @@ type PlaintextFromKind = K extends 'string' : K extends DateLikeCast ? Date : K extends 'json' - ? JsonValue + ? JsonDocument : never /** diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts index d2e817354..1e967897c 100644 --- a/packages/stack/src/eql/v3/drizzle/operators.ts +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -457,6 +457,22 @@ export function createEncryptionOperatorsV3( operator: string, ): Promise { requireNonNullOperand(ctx, value, operator) + // Reject the empty-object needle. `doc @> '{}'` holds for EVERY document + // (jsonb `{} ⊆ anything`), so `contains(col, {})` would silently return the + // whole table — the same whole-table footgun the bloom path guards against + // with `requireAnswerableNeedle`. An accidental empty filter is a bug, not a + // match-all request; callers wanting every row should omit the predicate. + if ( + value !== null && + typeof value === 'object' && + !Array.isArray(value) && + Object.keys(value).length === 0 + ) { + throw new EncryptionOperatorError( + `Operator "${operator}" cannot take an empty object needle on column "${ctx.columnName}": it matches every row. Pass a non-empty sub-object, or omit the predicate to select all rows.`, + { columnName: ctx.columnName, tableName: ctx.tableName, operator }, + ) + } const encryptQuery = client.encryptQuery?.bind(client) if (!encryptQuery) { throw operandFailure( diff --git a/packages/stack/src/eql/v3/index.ts b/packages/stack/src/eql/v3/index.ts index f02656b15..620132926 100644 --- a/packages/stack/src/eql/v3/index.ts +++ b/packages/stack/src/eql/v3/index.ts @@ -12,6 +12,7 @@ export type { AnyEncryptedV3Column, EncryptedV3TableColumn, EqlTypeForColumn, + JsonDocument, JsonValue, PlaintextForColumn, QueryCapabilities, diff --git a/skills/stash-encryption/SKILL.md b/skills/stash-encryption/SKILL.md index af6d5db03..599471043 100644 --- a/skills/stash-encryption/SKILL.md +++ b/skills/stash-encryption/SKILL.md @@ -644,7 +644,7 @@ Each factory in `types` maps 1:1 to a Postgres domain named `public.eql_v3_ Date: Sun, 12 Jul 2026 19:13:42 +1000 Subject: [PATCH 8/8] refactor(stack): review wording/hygiene for v3 JSON (no behaviour change) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the PR #621 review's wording/classification/test-hygiene points (design changes split to #622/#623): - Sweep `ste_vec` out of user-facing comments/test names/skills → "encrypted JSONB document" / `eql_v3_json`; keep `ste_vec` only where it is literally the protect-ffi config key (`indexes.ste_vec`) or the `SteVecDocument` payload type. - json-contains suite: drop the `makeEqlV3Column` wrapper (use the Drizzle `types.Json`), which lets both `as never` casts go (the pgTable and the bulkEncryptModels call) — the table is now properly typed; un-alias the Drizzle `eq`/`asc` imports. - operators unit: rename `storageDomains` → `nonScalarQueryDomains` and fix the comment — json is a QUERYABLE (containment) domain that answers no scalar op, not a storage-only one. - Reframe json's `deferred` marker across catalog/families/run-family-suite/ test: it means "not run by the scalar op-matrix" (covered by dedicated suites), NOT unimplemented — distinct from the ORE domains' superuser-only deferral. - Rewrite the opaque introspect comment as the explicit three-way case. - Skill: describe JSON as containment-only today; mark JSONPath selector as not-yet-implemented (tracked in #623); note `JsonDocument` (no top-level scalar). Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .../__tests__/drizzle-v3/operators.test.ts | 21 +++++++++------- .../stack/__tests__/test-kit-families.test.ts | 7 +++--- .../json-contains.integration.test.ts | 23 +++++++++--------- .../shared/json-crypto.integration.test.ts | 14 ++++++----- .../supabase/introspect.integration.test.ts | 11 +++++---- packages/stack/src/eql/v3/columns.ts | 21 ++++++++-------- .../stack/src/eql/v3/drizzle/operators.ts | 16 ++++++++----- packages/test-kit/src/catalog.ts | 16 +++++++------ packages/test-kit/src/families.ts | 10 ++++---- packages/test-kit/src/run-family-suite.ts | 15 ++++++------ skills/stash-encryption/SKILL.md | 24 ++++++++++++------- 11 files changed, 101 insertions(+), 77 deletions(-) diff --git a/packages/stack/__tests__/drizzle-v3/operators.test.ts b/packages/stack/__tests__/drizzle-v3/operators.test.ts index 06b79c2af..d96b7e801 100644 --- a/packages/stack/__tests__/drizzle-v3/operators.test.ts +++ b/packages/stack/__tests__/drizzle-v3/operators.test.ts @@ -115,18 +115,21 @@ const orderDomains = matrixEntries.filter( ([, spec]) => spec.indexes.ore || spec.indexes.ope, ) const matchDomains = matrixEntries.filter(([, spec]) => spec.indexes.match) -const storageDomains = matrixEntries.filter( +// Domains with NO scalar query index (unique/ore/ope/match), so `eq`/`ne`/ +// ordering all throw. This is the truly storage-only scalar domains PLUS +// `eql_v3_json` — json is a QUERYABLE domain (containment), it just answers no +// scalar op, so it lands here for the eq/order rejection checks. +const nonScalarQueryDomains = matrixEntries.filter( ([, spec]) => !spec.indexes.unique && !spec.indexes.ore && !spec.indexes.ope && !spec.indexes.match, ) -// Storage-only domains that also lack an ste_vec index: these reject `contains` -// outright. A json (`ste_vec`) column is storage-only for eq/order but DOES -// answer `contains` via `@>`, so it is excluded from the containment-rejection -// group and covered by its own positive assertion below. -const noContainmentDomains = storageDomains.filter( +// Of those, the ones that also reject `contains` — i.e. everything except json. +// json answers `contains` via `@>`, so it is excluded here and gets its own +// positive assertion in the JSON containment describe above. +const noContainmentDomains = nonScalarQueryDomains.filter( ([, spec]) => !spec.indexes.ste_vec, ) @@ -477,8 +480,8 @@ describe('createEncryptionOperatorsV3 - JSON containment', () => { }) }) -describe('createEncryptionOperatorsV3 - storage-only domains', () => { - it.each(storageDomains)('%s eq throws', async (eqlType, spec) => { +describe('createEncryptionOperatorsV3 - domains with no scalar query', () => { + it.each(nonScalarQueryDomains)('%s eq throws', async (eqlType, spec) => { const { ops } = setup() await expect( ops.eq(matrixColumn(eqlType), sampleFor(spec)), @@ -492,7 +495,7 @@ describe('createEncryptionOperatorsV3 - storage-only domains', () => { ).rejects.toBeInstanceOf(EncryptionOperatorError) }) - it.each(storageDomains)('%s asc throws synchronously', (eqlType) => { + it.each(nonScalarQueryDomains)('%s asc throws synchronously', (eqlType) => { const { ops } = setup() expect(() => ops.asc(matrixColumn(eqlType))).toThrow( EncryptionOperatorError, diff --git a/packages/stack/__tests__/test-kit-families.test.ts b/packages/stack/__tests__/test-kit-families.test.ts index 732a2e34e..d94a3c2d3 100644 --- a/packages/stack/__tests__/test-kit-families.test.ts +++ b/packages/stack/__tests__/test-kit-families.test.ts @@ -70,10 +70,11 @@ describe('test-kit families partition the v3 catalog', () => { 'text_ord_ore', 'timestamp_ord_ore', ]) - // ORE domains defer for the superuser-only opclass; json defers because - // ste_vec containment isn't in the scalar op matrix. + // ORE domains defer for the superuser-only opclass; json defers because it + // is queried by containment, not the scalar op-matrix (covered by dedicated + // suites — see catalog reason). for (const { bare, reason } of deferred) { - expect(reason).toMatch(bare === 'json' ? /ste_vec/ : /superuser-only/) + expect(reason).toMatch(bare === 'json' ? /containment/ : /superuser-only/) } }) diff --git a/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts b/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts index ad9640176..ce409bace 100644 --- a/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts +++ b/packages/stack/integration/drizzle-v3/json-contains.integration.test.ts @@ -2,23 +2,24 @@ * Live JSON containment for the v3 `types.Json()` column through the Drizzle * operators. Seeds encrypted JSONB documents and queries them with * `ops.contains(col, subObject)` — the same operator text search uses, but on a - * `json` column it emits the `@>` operator over the ste_vec document (json has - * no `eql_v3.contains` overload) with a `query_jsonb` needle from + * `json` column it emits the `@>` operator over the encrypted JSONB document + * (json has no `eql_v3.contains` overload) with a `query_jsonb` needle from * `encryptQuery` — asserting it returns exactly the rows whose document contains * the sub-object (jsonb `@>` semantics), and excludes the rest. */ import { databaseUrl, unwrapResult } from '@cipherstash/test-kit' -import { and, asc as drizzleAsc, eq as drizzleEq, type SQL } from 'drizzle-orm' +import { and, asc, eq, type SQL } from 'drizzle-orm' import { integer, pgTable, text } from 'drizzle-orm/pg-core' import { drizzle } from 'drizzle-orm/postgres-js' import postgres from 'postgres' import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { EncryptionV3, types } from '@/encryption/v3' +import { EncryptionV3 } from '@/encryption/v3' +import type { JsonDocument } from '@/eql/v3' import { createEncryptionOperatorsV3, extractEncryptionSchemaV3, + types, } from '@/eql/v3/drizzle' -import { makeEqlV3Column } from '@/eql/v3/drizzle/column' const sqlClient = postgres(databaseUrl(), { prepare: false }) @@ -29,13 +30,13 @@ const docTable = pgTable(TABLE_NAME, { id: integer('id').primaryKey().generatedAlwaysAsIdentity(), rowKey: text('row_key').notNull(), testRunId: text('test_run_id').notNull(), - doc: makeEqlV3Column(types.Json('doc')), -} as never) + doc: types.Json('doc'), +}) const schema = extractEncryptionSchemaV3(docTable) // Distinct documents so each containment query has a definite expected set. -const DOCS: Record> = { +const DOCS: Record = { ada: { user: 'ada@example.com', roles: ['admin', 'eng'], active: true }, grace: { user: 'grace@example.com', roles: ['eng'] }, zoe: { user: 'zoe@example.com', roles: ['ops'], active: false }, @@ -50,8 +51,8 @@ async function matching(condition: SQL): Promise { const rows = (await db .select({ rowKey: docTable.rowKey }) .from(docTable) - .where(and(drizzleEq(docTable.testRunId, RUN), condition)) - .orderBy(drizzleAsc(docTable.rowKey))) as SelectRow[] + .where(and(eq(docTable.testRunId, RUN), condition)) + .orderBy(asc(docTable.rowKey))) as SelectRow[] return rows.map((row) => row.rowKey) } @@ -77,7 +78,7 @@ beforeAll(async () => { doc, })) const encrypted = unwrapResult( - await client.bulkEncryptModels(rows as never, schema), + await client.bulkEncryptModels(rows, schema), ) as Array> await db.insert(docTable).values(encrypted as never) }, 120000) diff --git a/packages/stack/integration/shared/json-crypto.integration.test.ts b/packages/stack/integration/shared/json-crypto.integration.test.ts index f240d8754..77382a72a 100644 --- a/packages/stack/integration/shared/json-crypto.integration.test.ts +++ b/packages/stack/integration/shared/json-crypto.integration.test.ts @@ -1,8 +1,9 @@ /** * Live round-trip for the v3 `types.Json()` column — an encrypted JSONB document - * stored as an ste_vec `SteVecDocument`. Proves the new json column model - * encrypts and decrypts a real document through protect-ffi (no DB query here; - * containment is exercised by the Drizzle json suite). + * (`public.eql_v3_json`). Proves the new json column model encrypts and decrypts + * a real document through protect-ffi (no DB query here; containment is exercised + * by the Drizzle json suite). The stored payload is protect-ffi's + * `SteVecDocument`, carrying an `sv` array rather than a scalar `c` ciphertext. */ import { unwrapResult } from '@cipherstash/test-kit' import { beforeAll, describe, expect, it } from 'vitest' @@ -12,7 +13,7 @@ const docs = encryptedTable('v3_json_docs', { profile: types.Json('profile'), }) -describe('v3 typed client — JSON (ste_vec) round-trip', () => { +describe('v3 typed client — encrypted JSONB round-trip', () => { let client: Awaited>> beforeAll(async () => { @@ -30,7 +31,8 @@ describe('v3 typed client — JSON (ste_vec) round-trip', () => { const encrypted = unwrapResult( await client.encrypt(value, { table: docs, column: docs.profile }), ) - // An ste_vec document carries an `sv` array, not a scalar `c` ciphertext. + // An encrypted JSONB document carries an `sv` array, not a scalar `c` + // ciphertext (the stored shape is protect-ffi's `SteVecDocument`). expect(Array.isArray((encrypted as { sv?: unknown }).sv)).toBe(true) const decrypted = unwrapResult(await client.decrypt(encrypted)) @@ -46,7 +48,7 @@ describe('v3 typed client — JSON (ste_vec) round-trip', () => { // A JSON document is not only an object: `types.Json` (and the `JsonDocument` // plaintext type) also admit a top-level array and `null`. Pin both so a - // regression in ste_vec encoding of non-object roots turns a test red. + // regression in encrypted-JSONB encoding of non-object roots turns a test red. it.each([ ['array root', [1, 'two', { three: 3 }]], ['null root', null], diff --git a/packages/stack/integration/supabase/introspect.integration.test.ts b/packages/stack/integration/supabase/introspect.integration.test.ts index e49dcf4e9..11e70d16c 100644 --- a/packages/stack/integration/supabase/introspect.integration.test.ts +++ b/packages/stack/integration/supabase/introspect.integration.test.ts @@ -28,10 +28,13 @@ beforeAll(async () => { // as unmodelled — it is an ordinary plaintext passthrough. await sql.unsafe(`DROP DOMAIN IF EXISTS public.${USER_DOMAIN}`) await sql.unsafe(`CREATE DOMAIN public.${USER_DOMAIN} AS jsonb`) - // Mixed EQL columns: `score` is typed with an EQL domain the SDK has NO types - // factory for (unmodelled → reported); `doc` is `eql_v3_json`, which `types.Json` - // DOES model (so it must be synthesized, not reported); `own` is a user's plain - // jsonb domain (plaintext passthrough, never reported). + // This table exercises all three cases the introspector must tell apart: + // score → EQL domain (eql_v3_integer_ord_ope) the SDK has no factory for + // → "unmodelled", MUST be reported so the caller knows it can't query it + // doc → EQL domain (eql_v3_json) the SDK models via types.Json + // → modelled, MUST be synthesized and NOT reported + // own → a user's own plain jsonb domain with no EQL marker + // → plaintext passthrough, MUST never be reported await sql.unsafe(` CREATE TABLE ${UNMODELLED} ( id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, diff --git a/packages/stack/src/eql/v3/columns.ts b/packages/stack/src/eql/v3/columns.ts index 2e53e1ca0..fdcf56538 100644 --- a/packages/stack/src/eql/v3/columns.ts +++ b/packages/stack/src/eql/v3/columns.ts @@ -369,10 +369,10 @@ function indexesForCapabilities( } if (capabilities.searchableJson) { - // Encrypted-JSONB (ste_vec) index. `prefix: 'enabled'` is a sentinel the - // table's `build()` rewrites to `${tableName}/${columnName}` — the same - // scheme v2 `searchableJson` uses — so each document's selectors are scoped - // to their column. + // Encrypted-JSONB index (the protect-ffi config kind is named `ste_vec`). + // `prefix: 'enabled'` is a sentinel the table's `build()` rewrites to + // `${tableName}/${columnName}` — the same scheme v2 `searchableJson` uses — + // so each document's selectors are scoped to their column. // // `array_index_mode` indexes array elements by identity (`item`) and enables // `$[*]` wildcard selectors, but deliberately drops `position`: `@>` @@ -381,8 +381,8 @@ function indexesForCapabilities( // With positional terms emitted (v2 `searchableJson`'s `'all'`), the needle // for `x`-at-index-0 would miss a document holding `x` at index 1. // - // `mode: 'compat'` is REQUIRED for eql-3.0.0: `public.eql_v3_json` orders - // ste_vec entries by the CLLW-OPE `op` term, so the index must emit `op` + // `mode: 'compat'` is REQUIRED for eql-3.0.0: `public.eql_v3_json` orders the + // document's entries by the CLLW-OPE `op` term, so the index must emit `op` // (compat) terms. `'standard'` emits v2's CLLW-ORE `oc` terms, which the v3 // domain rejects at encrypt time. indexes.ste_vec = { @@ -633,10 +633,11 @@ const JSON_DOMAIN = { } as const /** - * Builder for a `public.eql_v3_json` column — an encrypted JSONB document, - * stored as an ste_vec `SteVecDocument` and queried by containment. The document - * (any {@link JsonValue}) round-trips through `encrypt`/`decrypt`; structural - * queries use the ste_vec index emitted by `build()`. + * Builder for a `public.eql_v3_json` column — an encrypted JSONB document + * (a {@link JsonDocument}: object, array, or null). It round-trips through + * `encrypt`/`decrypt`, and containment queries use the encrypted-JSONB index + * emitted by `build()`. (The stored payload is protect-ffi's `SteVecDocument`, + * and the config index kind is `ste_vec` — both protect-ffi names.) */ export class EncryptedJsonColumn extends EncryptedV3Column { constructor(columnName: string) { diff --git a/packages/stack/src/eql/v3/drizzle/operators.ts b/packages/stack/src/eql/v3/drizzle/operators.ts index 1e967897c..7adad1d9f 100644 --- a/packages/stack/src/eql/v3/drizzle/operators.ts +++ b/packages/stack/src/eql/v3/drizzle/operators.ts @@ -43,8 +43,11 @@ const MAX_IN_ARRAY_CONCURRENCY = 4 * * `bulkEncrypt` is optional so a `{ encrypt }`-only client stays valid; the * list operators fall back to bounded-concurrency single encryption without it. - * `encryptQuery` is likewise optional — only JSON containment (`@>`) needs it, - * to build a `query_jsonb` needle; the `contains()` branch guards its absence. + * `encryptQuery` is optional only because today it is used by a single operator + * — JSON containment (`@>`), which needs a ciphertext-free `query_jsonb` needle; + * the `contains()` branch guards its absence. Every OTHER operator still encrypts + * its operand with `encrypt` (a full storage envelope). Unifying all operands + * onto `encryptQuery` — after which it stops being optional — is tracked in #622. */ type OperandEncryptionClient = { encrypt( @@ -233,9 +236,10 @@ export function createEncryptionOperatorsV3( const ORDERING_INDEXES = ['ore', 'ope'] as const // `contains` answers two shapes: bloom free-text (`match`, a `text_search`/ // `text_match` column), which emits `eql_v3.contains(col, operand)`, and - // encrypted-JSONB containment (`ste_vec`, a `json` column), which emits the - // `@>` operator instead — json has no `eql_v3.contains` overload. The branch - // in `contains()` picks the right SQL by index kind. + // encrypted-JSONB containment (an `eql_v3_json` column, whose config carries + // the `ste_vec` index kind), which emits the `@>` operator instead — json has + // no `eql_v3.contains` overload. The branch in `contains()` picks the right SQL + // by index kind. const CONTAINMENT_INDEXES = ['match', 'ste_vec'] as const function applyOperationOptions( @@ -448,7 +452,7 @@ export function createEncryptionOperatorsV3( /** * Build a `query_jsonb` containment needle for a `json` column. Uses - * `encryptQuery` (not `encrypt`): the ste_vec query term carries no ciphertext + * `encryptQuery` (not `encrypt`): the JSON query term carries no ciphertext * and satisfies the `eql_v3.query_jsonb` CHECK the `@>` overload needs. */ async function encryptJsonContainmentTerm( diff --git a/packages/test-kit/src/catalog.ts b/packages/test-kit/src/catalog.ts index 0ee9c13bd..6ebe4287f 100644 --- a/packages/test-kit/src/catalog.ts +++ b/packages/test-kit/src/catalog.ts @@ -668,11 +668,13 @@ export const V3_MATRIX = { errorSamples: NUM_ERR, }, - // Encrypted JSONB document (ste_vec). DEFERRED from the scalar family driver: - // it is queried by containment/selector, not the eq/ord/match ops the oracle - // models, so it is covered by a dedicated json integration suite. The row - // still pins the built shape (`cast_as: 'json'` + the ste_vec index) and - // carries representative document samples. + // Encrypted JSONB document. Excluded from the SCALAR family driver (that is + // what `deferred` marks here) — NOT unimplemented: JSON is fully covered by + // its own live suites (`json-crypto`, `json-contains`). It is queried by + // containment (the `@>` operator), not the eq/ord/match ops the scalar oracle + // models, so `runFamilySuite` cannot exercise it. The row still pins the built + // shape (`cast_as: 'json'` + the encrypted-JSONB index) and carries + // representative document samples. 'public.eql_v3_json': { builder: types.Json, ColumnClass: EncryptedJsonColumn, @@ -695,7 +697,7 @@ export const V3_MATRIX = { { user: 'grace@example.com', roles: ['eng'] }, ], deferred: - 'json is queried by ste_vec containment/selector, not the scalar op ' + - 'matrix — covered by a dedicated json integration suite', + 'json is covered by dedicated live suites (json-crypto, json-contains), ' + + 'not the scalar op-matrix: it is queried by containment, not eq/ord/match', }, } as const satisfies Record diff --git a/packages/test-kit/src/families.ts b/packages/test-kit/src/families.ts index b0fa53c10..4ba86969a 100644 --- a/packages/test-kit/src/families.ts +++ b/packages/test-kit/src/families.ts @@ -47,10 +47,12 @@ const FAMILY_PREFIXES: Readonly> = { timestamp: ['timestamp'], text: ['text'], boolean: ['boolean'], - // The json family's only domain (`eql_v3_json`) is `deferred`: ste_vec - // containment doesn't fit the scalar oracle, so it's covered by a dedicated - // suite, not `runFamilySuite`. The prefix still owns it for coverage - // accounting. + // `json` is here only for coverage accounting — its one domain (`eql_v3_json`) + // is marked `deferred`, meaning "not run by the scalar op-matrix", because JSON + // is queried by containment (`@>`), not the eq/ord/match ops the oracle models. + // It is NOT unimplemented: dedicated live suites (`json-crypto`, `json-contains`) + // cover it. (Contrast the ORE domains, also `deferred` but because their opclass + // is superuser-only and cannot run on managed Postgres.) json: ['json'], } diff --git a/packages/test-kit/src/run-family-suite.ts b/packages/test-kit/src/run-family-suite.ts index c17cb3278..524df7ecf 100644 --- a/packages/test-kit/src/run-family-suite.ts +++ b/packages/test-kit/src/run-family-suite.ts @@ -151,14 +151,13 @@ export function runFamilySuite( const adapter = makeAdapter() const domains = domainsForFamily(family) - // A family whose every domain is deferred (e.g. `json`: ste_vec containment - // and selector queries don't fit the scalar oracle this driver runs). It - // carries no op-matrix coverage here — its real coverage lives in dedicated - // suites (`json-crypto`, `json-contains`). Emit one assertion documenting the - // deferral so the family stays visible and is not a silent skip, rather than - // letting `planTable` throw on an empty column set. A family with NO domains - // at all (covered or deferred) is a wiring bug and still falls through to that - // throw. + // A family whose every domain is deferred (e.g. `json`: containment queries + // don't fit the scalar oracle this driver runs). It carries no op-matrix + // coverage here — its real coverage lives in dedicated suites (`json-crypto`, + // `json-contains`). Emit one assertion documenting the deferral so the family + // stays visible and is not a silent skip, rather than letting `planTable` throw + // on an empty column set. A family with NO domains at all (covered or deferred) + // is a wiring bug and still falls through to that throw. if (domains.length === 0) { const deferred = deferredForFamily(family) if (deferred.length > 0) { diff --git a/skills/stash-encryption/SKILL.md b/skills/stash-encryption/SKILL.md index 599471043..794e2116b 100644 --- a/skills/stash-encryption/SKILL.md +++ b/skills/stash-encryption/SKILL.md @@ -632,7 +632,7 @@ Each factory in `types` maps 1:1 to a Postgres domain named `public.eql_v3_` semantics). Array containment is a subset test regardless of - element position: `{ roles: ['admin'] }` matches any document whose `roles` - array includes `admin`. -- **JSONPath selector** — pass a `'$.path'` string to query into the document. +**Containment** is the supported query today. Pass a sub-object or sub-array to +`encryptQuery` with `queryType: 'searchableJson'`; it matches documents that +contain the needle (jsonb `@>` semantics). Array containment is a subset test +regardless of element position — `{ roles: ['admin'] }` matches any document +whose `roles` array includes `admin`. ```typescript const events = encryptedTable("events", { metadata: types.Json("metadata") }) @@ -689,6 +689,12 @@ Through the Drizzle v3 integration this is `ops.contains(col, subObject)` — se the `stash-drizzle` skill. `types.Json` carries no equality or ordering, so `eq` / `gt` / `asc` on it throw. +> **Not yet implemented:** JSONPath selector-with-constraint queries +> (`metadata->'plan' = $1`, `metadata->'age' > $1`) — a distinct third pattern +> the `eql_v3_json` domain supports at the SQL level (`->` / `->>`). Neither the +> query operator nor the selector-string needle typing is wired up yet; tracked +> in [#623](https://github.com/cipherstash/stack/issues/623). + ### Strongly-Typed Client: `EncryptionV3` `EncryptionV3` from `@cipherstash/stack/v3` returns a `TypedEncryptionClient` whose method signatures are derived from your schemas — wrong-typed plaintext is rejected at compile time, and query methods only accept queryable columns with `queryType` constrained to the column's capabilities: