diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 8f97410644..f635f6a178 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -52,6 +52,15 @@ jobs: - name: build run: yarn build + - name: seed app_user + run: | + psql -f ./bootstrap-roles.sql postgres + env: + PGHOST: pg_db + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: password + - name: launchql/migrate run: cd ./packages/migrate && yarn test @@ -72,6 +81,17 @@ jobs: yarn test env: PGHOST: pg_db + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: password + + - name: pgsql-test + run: | + cd ./packages/pgsql-test + yarn test + env: + PGHOST: pg_db + PGPORT: 5432 PGUSER: postgres PGPASSWORD: password diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 7c13c54abf..556f8477c8 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -16,6 +16,7 @@ import plan from './commands/plan'; import _export from './commands/export'; import _package from './commands/package'; import kill from './commands/kill'; +import { teardownPgPools } from '@launchql/server-utils'; // Command map const commandMap: Record = { @@ -82,5 +83,7 @@ export const commands = async (argv: Partial, prompter: Inquirerer, await commandFn(newArgv, prompter, options); prompter.close(); + + await teardownPgPools(); return argv; }; diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts index 922dd7a081..0db5f84ea5 100644 --- a/packages/cli/src/commands/export.ts +++ b/packages/cli/src/commands/export.ts @@ -1,7 +1,6 @@ import { CLIOptions, Inquirerer, OptionValue } from 'inquirerer'; import { LaunchQLProject, exportMigrations } from '@launchql/migrate'; import { getEnvOptions } from '@launchql/types'; -import chalk from 'chalk'; import { resolve } from 'path'; import { execSync } from 'child_process'; import { getRootPgPool } from '@launchql/server-utils'; @@ -20,10 +19,8 @@ export default async ( project.ensureWorkspace(); const options = getEnvOptions(); - const { pg } = options; const db = await getRootPgPool({ - ...pg, database: 'postgres' }); @@ -46,7 +43,6 @@ export default async ( const dbname = databases.filter(d=>d.selected).map(d=>d.value)[0]; const selectedDb = await getRootPgPool({ - ...pg, database: dbname }); diff --git a/packages/cli/src/commands/kill.ts b/packages/cli/src/commands/kill.ts index c4664fdf40..da9a0ab3e2 100644 --- a/packages/cli/src/commands/kill.ts +++ b/packages/cli/src/commands/kill.ts @@ -1,5 +1,4 @@ import { CLIOptions, Inquirerer, OptionValue } from 'inquirerer'; -import { getEnvOptions } from '@launchql/types'; import { getRootPgPool } from '@launchql/server-utils'; import chalk from 'chalk'; @@ -8,11 +7,8 @@ export default async ( prompter: Inquirerer, _options: CLIOptions ) => { - const options = getEnvOptions(); - const { pg } = options; const db = await getRootPgPool({ - ...pg, database: 'postgres' }); diff --git a/packages/explorer/src/server.ts b/packages/explorer/src/server.ts index f0f43857db..cd27ff07cd 100644 --- a/packages/explorer/src/server.ts +++ b/packages/explorer/src/server.ts @@ -12,11 +12,11 @@ import { } from '@launchql/server-utils'; import { printSchemas, printDatabases } from './render'; import { getGraphileSettings } from './settings'; -import { LaunchQLOptions } from '@launchql/types'; -import { getMergedOptions } from '@launchql/types'; +import { getPgEnvOptions, LaunchQLOptions } from '@launchql/types'; +import { getEnvOptions } from '@launchql/types'; export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { - const opts = getMergedOptions(rawOpts); + const opts = getEnvOptions(rawOpts); const { pg, @@ -39,10 +39,12 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { graphiqlRoute: '/graphiql' }; - const pgPool = getRootPgPool({ - ...opts.pg, - database: dbname - }); + const pgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: dbname + })); + const handler = postgraphile(pgPool, schemaname, settings); const obj = { @@ -67,10 +69,12 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { if (req.urlDomains?.subdomains.length === 1) { const [dbName] = req.urlDomains.subdomains; try { - const pgPool = getRootPgPool({ - ...opts.pg, - database: dbName - }); + const pgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: dbName + })); + const results = await pgPool.query(` SELECT s.nspname AS table_schema FROM pg_catalog.pg_namespace s @@ -103,11 +107,15 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { if (req.urlDomains?.subdomains.length === 2) { const [, dbName] = req.urlDomains.subdomains; try { - const pgPool = getRootPgPool({ - ...opts.pg, - database: dbName - }); + + const pgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: dbName + })); + await pgPool.query('SELECT 1;'); + } catch (e: any) { if (e.message?.match(/does not exist/)) { res.status(404).send('DB Not found'); @@ -150,10 +158,12 @@ export const LaunchQLExplorer = (rawOpts: LaunchQLOptions = {}): Express => { app.use(async (req: Request, res: Response, next: NextFunction) => { if (req.urlDomains?.subdomains.length === 0) { try { - const rootPgPool = getRootPgPool({ - ...opts.pg, - database: opts.pg.user // is this to get postgres? - }); + const rootPgPool = getRootPgPool( + getPgEnvOptions({ + ...opts.pg, + database: opts.pg.user // is this to get postgres? + })); + const results = await rootPgPool.query(` SELECT * FROM pg_catalog.pg_database WHERE datistemplate = FALSE AND datname != 'postgres' AND datname !~ '^pg_' diff --git a/packages/explorer/src/settings.ts b/packages/explorer/src/settings.ts index ce27bd7ad2..1f1a503818 100644 --- a/packages/explorer/src/settings.ts +++ b/packages/explorer/src/settings.ts @@ -1,10 +1,10 @@ import { PostGraphileOptions } from 'postgraphile'; import { getGraphileSettings as getSettings } from '@launchql/graphile-settings'; import { LaunchQLOptions } from '@launchql/types'; -import { getMergedOptions } from '@launchql/types'; +import { getEnvOptions } from '@launchql/types'; export const getGraphileSettings = (rawOpts: LaunchQLOptions): PostGraphileOptions => { - const opts = getMergedOptions(rawOpts); + const opts = getEnvOptions(rawOpts); const baseOptions = getSettings(opts); diff --git a/packages/graphile-settings/package.json b/packages/graphile-settings/package.json index fd9deb6bb5..81ecf81540 100644 --- a/packages/graphile-settings/package.json +++ b/packages/graphile-settings/package.json @@ -38,7 +38,6 @@ "@pyramation/postgis": "^0.1.1", "@pyramation/postgraphile-plugin-fulltext-filter": "^2.0.0", "cors": "^2.8.5", - "envalid": "^8.0.0", "express": "^5.1.0", "graphile-build": "^4.14.1", "graphile-i18n": "^0.0.3", diff --git a/packages/graphile-settings/src/index.ts b/packages/graphile-settings/src/index.ts index f2b6b7c6d9..872bf3b97c 100644 --- a/packages/graphile-settings/src/index.ts +++ b/packages/graphile-settings/src/index.ts @@ -23,10 +23,10 @@ import LqlTypesPlugin from './plugins/types'; import { Uploader } from './resolvers/upload'; import { PostGraphileOptions } from 'postgraphile'; import { LaunchQLOptions } from '@launchql/types'; -import { getMergedOptions } from '@launchql/types'; +import { getEnvOptions } from '@launchql/types'; export const getGraphileSettings = (rawOpts: LaunchQLOptions): PostGraphileOptions => { - const opts = getMergedOptions(rawOpts); + const opts = getEnvOptions(rawOpts); const { server, diff --git a/packages/migrate/src/deploy.ts b/packages/migrate/src/deploy.ts index bd67041a75..e551a63e04 100644 --- a/packages/migrate/src/deploy.ts +++ b/packages/migrate/src/deploy.ts @@ -1,7 +1,7 @@ import { resolve } from 'path'; import chalk from 'chalk'; -import { LaunchQLOptions } from '@launchql/types'; +import { getSpawnEnvWithPg, LaunchQLOptions } from '@launchql/types'; import { getRootPgPool } from '@launchql/server-utils'; import { LaunchQLProject } from './class/launchql'; import { spawn } from 'child_process'; @@ -53,7 +53,7 @@ export const deploy = async ( const child = spawn('sqitch', ['deploy', `db:pg:${database}`], { cwd: modulePath, - env: { ...process.env } + env: getSpawnEnvWithPg(opts.pg) }); const exitCode: number = await new Promise((resolve, reject) => { @@ -91,6 +91,6 @@ export const deploy = async ( } console.log(chalk.green(`\nβœ… Deployment complete for ${chalk.bold(name)}.\n`)); - await pgPool.end(); + // await pgPool.end(); return extensions; }; diff --git a/packages/migrate/src/export-migrations.ts b/packages/migrate/src/export-migrations.ts index ed85710326..7238340c96 100644 --- a/packages/migrate/src/export-migrations.ts +++ b/packages/migrate/src/export-migrations.ts @@ -10,149 +10,149 @@ import { SqitchRow, writeSqitchFiles, writeSqitchPlan } from './sqitch'; import { LaunchQLProject } from './class/launchql'; interface ExportMigrationsToDiskOptions { - project: LaunchQLProject; - options: LaunchQLOptions; - database: string; - databaseId: string; - author: string; - outdir: string; - schema_names: string[]; - extensionName?: string; - metaExtensionName: string; + project: LaunchQLProject; + options: LaunchQLOptions; + database: string; + databaseId: string; + author: string; + outdir: string; + schema_names: string[]; + extensionName?: string; + metaExtensionName: string; } interface ExportOptions { - project: LaunchQLProject; - options: LaunchQLOptions; - dbInfo: { - dbname: string; - database_ids: string[]; - }; - author: string; - outdir: string; - schema_names: string[]; - extensionName?: string; - metaExtensionName: string; + project: LaunchQLProject; + options: LaunchQLOptions; + dbInfo: { + dbname: string; + database_ids: string[]; + }; + author: string; + outdir: string; + schema_names: string[]; + extensionName?: string; + metaExtensionName: string; } const exportMigrationsToDisk = async ({ - project, - options, - database, - databaseId, - author, - outdir, - schema_names, - extensionName, - metaExtensionName + project, + options, + database, + databaseId, + author, + outdir, + schema_names, + extensionName, + metaExtensionName }: ExportMigrationsToDiskOptions): Promise => { - outdir = outdir + '/'; + outdir = outdir + '/'; - const pgPool = getRootPgPool({ - ...options.pg, - database - }); + const pgPool = getRootPgPool({ + ...options.pg, + database + }); - const db = await pgPool.query( - `select * from collections_public.database where id=$1`, - [databaseId] - ); + const db = await pgPool.query( + `select * from collections_public.database where id=$1`, + [databaseId] + ); - const schemas = await pgPool.query( - `select * from collections_public.schema where database_id=$1`, - [databaseId] - ); + const schemas = await pgPool.query( + `select * from collections_public.schema where database_id=$1`, + [databaseId] + ); - if (!db?.rows?.length) { - console.log('NO DATABASES.'); - return; - } + if (!db?.rows?.length) { + console.log('NO DATABASES.'); + return; + } - if (!schemas?.rows?.length) { - console.log('NO SCHEMAS.'); - return; - } + if (!schemas?.rows?.length) { + console.log('NO SCHEMAS.'); + return; + } + + const name = extensionName || db.rows[0].name; - const name = extensionName || db.rows[0].name; + const { replace, replacer } = makeReplacer({ + schemas: schemas.rows.filter((schema: any) => + schema_names.includes(schema.schema_name) + ), + name + }); + + const results = await pgPool.query( + `select * from db_migrate.sql_actions order by id` + ); - const { replace, replacer } = makeReplacer({ - schemas: schemas.rows.filter((schema: any) => - schema_names.includes(schema.schema_name) - ), - name + const opts = { + name, + replace, + replacer, + outdir, + author + }; + + if (results?.rows?.length > 0) { + await preparePackage({ + project, + author, + outdir, + name, + extensions: [ + 'plpgsql', + 'uuid-ossp', + 'citext', + 'pgcrypto', + 'btree_gist', + 'postgis', + 'hstore', + 'db_meta', + 'launchql-inflection', + 'launchql-uuid', + 'launchql-utils', + 'launchql-ext-jobs', + 'launchql-jwt-claims', + 'launchql-stamps', + 'launchql-base32', + 'launchql-totp', + 'launchql-ext-types', + 'launchql-ext-default-roles' + ] + }); + + writeSqitchPlan(results.rows, opts); + writeSqitchFiles(results.rows, opts); + + let meta = await exportMeta({ + opts: options, + dbname: database, + database_id: databaseId + }); + + meta = replacer(meta); + + await preparePackage({ + project, + author, + outdir, + extensions: ['plpgsql', 'db_meta', 'db_meta_modules'], + name: metaExtensionName + }); + + const metaReplacer = makeReplacer({ + schemas: schemas.rows.filter((schema: any) => + schema_names.includes(schema.schema_name) + ), + name: metaExtensionName }); - const results = await pgPool.query( - `select * from db_migrate.sql_actions order by id` - ); - - const opts = { - name, - replace, - replacer, - outdir, - author - }; - - if (results?.rows?.length > 0) { - await preparePackage({ - project, - author, - outdir, - name, - extensions: [ - 'plpgsql', - 'uuid-ossp', - 'citext', - 'pgcrypto', - 'btree_gist', - 'postgis', - 'hstore', - 'db_meta', - 'launchql-inflection', - 'launchql-uuid', - 'launchql-utils', - 'launchql-ext-jobs', - 'launchql-jwt-claims', - 'launchql-stamps', - 'launchql-base32', - 'launchql-totp', - 'launchql-ext-types', - 'launchql-ext-default-roles' - ] - }); - - writeSqitchPlan(results.rows, opts); - writeSqitchFiles(results.rows, opts); - - let meta = await exportMeta({ - opts: options, - dbname: database, - database_id: databaseId - }); - - meta = replacer(meta); - - await preparePackage({ - project, - author, - outdir, - extensions: ['plpgsql', 'db_meta', 'db_meta_modules'], - name: metaExtensionName - }); - - const metaReplacer = makeReplacer({ - schemas: schemas.rows.filter((schema: any) => - schema_names.includes(schema.schema_name) - ), - name: metaExtensionName - }); - - const metaPackage: SqitchRow[] = [ - { - deps: [], - deploy: 'migrate/meta', - content: `SET session_replication_role TO replica; + const metaPackage: SqitchRow[] = [ + { + deps: [], + deploy: 'migrate/meta', + content: `SET session_replication_role TO replica; -- using replica in case we are deploying triggers to collections_public -- unaccent, postgis affected and require grants @@ -178,43 +178,43 @@ UPDATE meta_public.sites SET session_replication_role TO DEFAULT; ` - } - ]; + } + ]; - opts.replacer = metaReplacer.replacer; - opts.name = metaExtensionName; + opts.replacer = metaReplacer.replacer; + opts.name = metaExtensionName; - writeSqitchPlan(metaPackage, opts); - writeSqitchFiles(metaPackage, opts); - } + writeSqitchPlan(metaPackage, opts); + writeSqitchFiles(metaPackage, opts); + } - pgPool.end(); + pgPool.end(); }; export const exportMigrations = async ({ - project, - options, - dbInfo, - author, - outdir, - schema_names, - extensionName, - metaExtensionName + project, + options, + dbInfo, + author, + outdir, + schema_names, + extensionName, + metaExtensionName }: ExportOptions): Promise => { - for (let v = 0; v < dbInfo.database_ids.length; v++) { - const databaseId = dbInfo.database_ids[v]; - await exportMigrationsToDisk({ - project, - options, - extensionName, - metaExtensionName, - database: dbInfo.dbname, - databaseId, - schema_names, - author, - outdir - }); - } + for (let v = 0; v < dbInfo.database_ids.length; v++) { + const databaseId = dbInfo.database_ids[v]; + await exportMigrationsToDisk({ + project, + options, + extensionName, + metaExtensionName, + database: dbInfo.dbname, + databaseId, + schema_names, + author, + outdir + }); + } }; @@ -259,10 +259,10 @@ const preparePackage = async ({ const plan = glob(path.join(sqitchDir, 'sqitch.plan')); if (!plan.length) { project.initModule({ - name, - description: name, - author, - extensions, + name, + description: name, + author, + extensions, }); } else { rmSync(path.resolve(sqitchDir, 'deploy'), { recursive: true, force: true }); diff --git a/packages/migrate/src/stream-sql.ts b/packages/migrate/src/stream-sql.ts index 10bcd0fe86..c95cc06ca8 100644 --- a/packages/migrate/src/stream-sql.ts +++ b/packages/migrate/src/stream-sql.ts @@ -1,6 +1,6 @@ +import { getSpawnEnvWithPg } from '@launchql/types'; import { spawn } from 'child_process'; import { Readable } from 'stream'; -import { env } from 'process'; interface PgConfig { user: string; @@ -39,16 +39,8 @@ export async function streamSql(config: PgConfig, sql: string): Promise { return new Promise((resolve, reject) => { const sqlStream = stringToStream(sql); - // TODO set env vars! const proc = spawn('psql', args, { - env: { - ...env, - PGUSER: config.user, - PGHOST: config.host, - PGDATABASE: config.database, - PGPASSWORD: config.password, - // PGPORT: config.port - } + env: getSpawnEnvWithPg(config) }); sqlStream.pipe(proc.stdin); diff --git a/packages/pgsql-test/README.md b/packages/pgsql-test/README.md new file mode 100644 index 0000000000..7cd8961991 --- /dev/null +++ b/packages/pgsql-test/README.md @@ -0,0 +1,139 @@ +# pgsql-test + +

+ +

+ +

+ + + + + + + + + +

+ +## Install + +```sh +npm install pgsql-test +``` +--- + +## Features + +* πŸ§ͺ Auto-generated test databases with `UUID` suffix +* πŸ”„ Per-test isolation using transactions and savepoints +* πŸ›‘οΈ Role-based context for RLS testing +* 🧹 Easy teardown and cleanup +* 🧰 Designed for `Jest`, `Mocha`, or any async test runner + +--- + +## How to Use + +`pgsql-test` provides an isolated PostgreSQL testing environment with per-test transaction rollback, ideal for integration tests involving SQL, roles, or GraphQL (e.g., with PostGraphile). + +### Basic Example + +```ts +import { PgTestClient, getConnections } from 'pgsql-test'; + +let conn: PgTestClient; +let db: PgTestClient; +let teardown: () => Promise; + +beforeAll(async () => { + ({ conn, db, teardown } = await getConnections()); + + await db.query(` + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL + ); + CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES users(id), + content TEXT NOT NULL + ); + `); + + await db.query(` + INSERT INTO users (name) VALUES ('Alice'), ('Bob'); + INSERT INTO posts (user_id, content) VALUES + (1, 'Hello world!'), + (2, 'Graphile is cool!'); + `); +}); + +afterAll(async () => { + await teardown(); +}); + +beforeEach(async () => { + await db.beforeEach(); // Starts transaction + SAVEPOINT +}); + +afterEach(async () => { + await db.afterEach(); // Rolls back to SAVEPOINT +}); + +test('user count starts at 2', async () => { + const res = await db.query('SELECT COUNT(*) FROM users'); + expect(res.rows[0].count).toBe('2'); +}); +``` + +--- + +## Role-Based Contexts + +You can simulate different PostgreSQL roles for RLS and permission testing. + +```ts +describe('authenticated role', () => { + beforeEach(async () => { + conn.setContext({ role: 'authenticated' }); + await conn.beforeEach(); + }); + + afterEach(async () => { + await conn.afterEach(); + }); + + it('runs as authenticated', async () => { + const result = await conn.query(`SELECT current_setting('role', true) AS role`); + expect(result.rows[0].role).toBe('authenticated'); + }); +}); +``` + +--- + +## Environment Overrides + +`pgsql-test` respects the following env vars for DB connectivity: + +* `PGHOST` +* `PGPORT` +* `PGUSER` +* `PGPASSWORD` + +Override them in your test runner or CI config: + +```yaml +env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: password +``` + +## Disclaimer + +AS DESCRIBED IN THE LICENSES, THE SOFTWARE IS PROVIDED β€œAS IS”, AT YOUR OWN RISK, AND WITHOUT WARRANTIES OF ANY KIND. + +No developer or entity involved in creating this software will be liable for any claims or damages whatsoever associated with your use, inability to use, or your interaction with other users of the code, including any direct, indirect, incidental, special, exemplary, punitive or consequential damages, or loss of profits, cryptocurrencies, tokens, or anything else of value. diff --git a/packages/pgsql-test/__tests__/postgres-test.connections.test.ts b/packages/pgsql-test/__tests__/postgres-test.connections.test.ts new file mode 100644 index 0000000000..90d5195cd2 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.connections.test.ts @@ -0,0 +1,52 @@ +import { getConnections } from '../src/connect'; +import { PgTestClient } from '../src/test-client'; + +let db: PgTestClient; +let teardown: () => Promise; + +beforeAll(async () => { + ({ db, teardown } = await getConnections()); +}); + +afterAll(async () => { + await teardown(); +}); + +describe('anonymous', () => { + beforeEach(async () => { + await db.beforeEach(); + }); + + afterEach(async () => { + await db.afterEach(); + }); + + it('runs under anonymous context', async () => { + const result = await db.query('SELECT current_setting(\'role\', true) AS role'); + expect(result.rows[0].role).toBe('anonymous'); + }); +}); + +describe('authenticated', () => { + beforeEach(async () => { + db.setContext({ + role: 'authenticated', + 'jwt.claims.user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36', + 'jwt.claims.ip_address': '127.0.0.1', + 'jwt.claims.database_id': 'jwt.database_id', + 'jwt.claims.user_id': 'jwt.user_id' + }); + await db.beforeEach(); + }); + + afterEach(async () => { + await db.afterEach(); + }); + + it('runs under authenticated context', async () => { + const role = await db.query('SELECT current_setting(\'role\', true) AS role'); + expect(role.rows[0].role).toBe('authenticated'); + const ip = await db.query('SELECT current_setting(\'jwt.claims.ip_address\', true) AS role'); + expect(ip.rows[0].role).toBe('127.0.0.1'); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts b/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts new file mode 100644 index 0000000000..234c7fb649 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.deploy-fast.test.ts @@ -0,0 +1,82 @@ +import path, { resolve } from 'path'; + +import { PgTestClient } from '../src/test-client'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); + +let pg: PgTestClient; +let teardown: () => Promise; + +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; +let totalStart: number; + +beforeAll(async () => { + totalStart = Date.now(); + + ({ pg, teardown } = await getConnections({}, { + cwd: resolve(__dirname + '/../../../__fixtures__/sqitch/simple/packages/my-third') + })); + + const admin = new DbAdmin(pg.config); + admin.loadSql(sql('test.sql'), pg.config.database); + admin.loadSql(sql('roles.sql'), pg.config.database); + + usedDbNames.push(pg.config.database); +}); + +beforeEach(async () => { + await pg.beforeEach(); + start = Date.now(); +}); + +afterEach(async () => { + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); + + await pg.afterEach(); +}); + +afterAll(async () => { + await teardown(); + + const avg = + testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const totalTime = Date.now() - totalStart; + + const summaryLines = [ + `πŸ§ͺ LaunchQL DeployFast DB Benchmark`, + `πŸ“‚ DB Used: ${usedDbNames[0]}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); +}); + +describe('LaunchQL DeployFast DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.deploy-sqitch.test.ts b/packages/pgsql-test/__tests__/postgres-test.deploy-sqitch.test.ts new file mode 100644 index 0000000000..6512ed16bf --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.deploy-sqitch.test.ts @@ -0,0 +1,89 @@ +import path, { resolve } from 'path'; + +import { PgTestClient } from '../src/test-client'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; +import { getRootPgPool } from '@launchql/server-utils'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); + +let pg: PgTestClient; +let teardown: () => Promise; + +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; +let totalStart: number; + +beforeAll(async () => { + totalStart = Date.now(); + + ({ pg, teardown } = await getConnections({}, { + deployFast: false, + cwd: resolve(__dirname + '/../../../__fixtures__/sqitch/simple/packages/my-third') + })); + + const admin = new DbAdmin(pg.config); + admin.loadSql(sql('test.sql'), pg.config.database); + admin.loadSql(sql('roles.sql'), pg.config.database); + + usedDbNames.push(pg.config.database); +}); + +beforeEach(async () => { + await pg.beforeEach(); + start = Date.now(); +}); + +afterEach(async () => { + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); + + await pg.afterEach(); +}); + +afterAll(async () => { + + // clear this DB first! so teardown() doesn't choke...a + const pgPool = getRootPgPool({ ...pg.config, database: pg.config.database }); + await pgPool.end(); + + await teardown(); + + const avg = + testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const totalTime = Date.now() - totalStart; + + const summaryLines = [ + `πŸ§ͺ LaunchQL Deploy (sqtich) DB Benchmark`, + `πŸ“‚ DB Used: ${usedDbNames[0]}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); +}); + +describe('LaunchQL Deploy (sqitch) DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.grants.test.ts b/packages/pgsql-test/__tests__/postgres-test.grants.test.ts new file mode 100644 index 0000000000..fdc240adc7 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.grants.test.ts @@ -0,0 +1,49 @@ +import { randomUUID } from 'crypto'; +import { resolve } from 'path'; +import { PgTestClient } from '../src/test-client'; +import { PgConfig } from '@launchql/types'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; +import { PgTestConnector } from '../src/manager'; + +let db: PgTestClient; +let pg: PgTestClient; +let teardown: () => Promise; + +const TEST_DB_NAME = `postgres_test_${randomUUID()}`; +const sqlPath = (file: string) => resolve(__dirname, '../sql', file); + +/** + * Optionally load seed SQL into the base template before forking connections. + */ +function setupBaseDatabase(config: PgConfig) { + const admin = new DbAdmin(config); + admin.loadSql(sqlPath('roles.sql'), config.database); + admin.loadSql(sqlPath('test.sql'), config.database); +} + +beforeAll(async () => { + // Create test DB and clients + ({ db, pg, teardown } = await getConnections({ + database: TEST_DB_NAME + })); + + // Optionally preload schema/data before tests + setupBaseDatabase(pg.config); +}); + +afterAll(async () => { + await teardown(); // closes manager and drops DB if needed +}); + +describe('Database Setup', () => { + it('has valid connection (pg)', async () => { + const result = await pg.query('SELECT 1 AS ok'); + expect(result.rows[0].ok).toBe(1); + }); + + it('has valid connection (db)', async () => { + const result = await db.query('SELECT 2 AS ok'); + expect(result.rows[0].ok).toBe(2); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.records.test.ts b/packages/pgsql-test/__tests__/postgres-test.records.test.ts new file mode 100644 index 0000000000..c3830fded2 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.records.test.ts @@ -0,0 +1,66 @@ +import { getPgEnvOptions } from '@launchql/types'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; +import { PgTestClient } from '../src/test-client'; + +let pg: PgTestClient; +let teardown: () => Promise; + +const setupSchemaSQL = ` + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL + ); + + CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES users(id), + content TEXT NOT NULL + ); +`; + +const seedDataSQL = ` + INSERT INTO users (name) VALUES ('Alice'), ('Bob'); + INSERT INTO posts (user_id, content) VALUES + (1, 'Hello world!'), + (2, 'Graphile is cool!'); +`; + +beforeAll(async () => { + ({ pg, teardown } = await getConnections()); + // create schema + seed *once* + await pg.query(setupSchemaSQL); + await pg.query(seedDataSQL); +}); + +afterAll(async () => { + await teardown(); +}); + +describe('Postgres Test Framework', () => { + beforeEach(async () => { + await pg.beforeEach(); // BEGIN + SAVEPOINT + }); + + afterEach(async () => { + await pg.afterEach(); // ROLLBACK TO SAVEPOINT + COMMIT + }); + + it('should have 2 users initially', async () => { + const { rows } = await pg.query('SELECT COUNT(*) FROM users'); + expect(rows[0].count).toBe('2'); + }); + + it('inserts a user but rollback leaves baseline intact', async () => { + await pg.query(`INSERT INTO users (name) VALUES ('Carol')`); + let res = await pg.query('SELECT COUNT(*) FROM users'); + expect(res.rows[0].count).toBe('3'); // inside this tx + // after rollback... the next test, we’ll still see 2 + }); + + it('still sees 2 users after previous insert test', async () => { + const { rows } = await pg.query('SELECT COUNT(*) FROM users'); + expect(rows[0].count).toBe('2'); + }); + +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts b/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts new file mode 100644 index 0000000000..b42f8088b3 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.rollbacks.test.ts @@ -0,0 +1,79 @@ +import path from 'path'; + +import { PgTestClient } from '../src/test-client'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); + +let pg: PgTestClient; +let teardown: () => Promise; + +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; +let totalStart: number; + +beforeAll(async () => { + totalStart = Date.now(); + + ({ pg, teardown } = await getConnections()); + + const admin = new DbAdmin(pg.config); + admin.loadSql(sql('test.sql'), pg.config.database); + admin.loadSql(sql('roles.sql'), pg.config.database); + + usedDbNames.push(pg.config.database); +}); + +beforeEach(async () => { + await pg.beforeEach(); + start = Date.now(); +}); + +afterEach(async () => { + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); + + await pg.afterEach(); +}); + +afterAll(async () => { + await teardown(); + + const totalTime = Date.now() - totalStart; + const avg = + testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const summaryLines = [ + `πŸ§ͺ Rollback DB Benchmark`, + `πŸ“‚ DB Used: ${usedDbNames[0]}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); +}); + +describe('Rollback DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); + }); +}); diff --git a/packages/pgsql-test/__tests__/postgres-test.template.test.ts b/packages/pgsql-test/__tests__/postgres-test.template.test.ts new file mode 100644 index 0000000000..e508e01d56 --- /dev/null +++ b/packages/pgsql-test/__tests__/postgres-test.template.test.ts @@ -0,0 +1,95 @@ +import path from 'path'; +import { randomUUID } from 'crypto'; +import { getPgEnvOptions } from '@launchql/types'; + +import { PgTestClient } from '../src/test-client'; +import { DbAdmin } from '../src/admin'; +import { getConnections } from '../src/connect'; + +const sql = (file: string) => path.resolve(__dirname, '../sql', file); + +let db: PgTestClient; +let pg: PgTestClient; +let teardown: () => Promise; + +const TEMPLATE_NAME = 'test_template'; +const SEED_DB_TO_CREATE_TEMPLATE = `postgres_test_${randomUUID()}`; + +const usedDbNames: string[] = []; +const testResults: { name: string; time: number }[] = []; + +let start: number; +let totalStart: number; + +function setupTemplateDatabase(): void { + const templateConfig = getPgEnvOptions({ + database: SEED_DB_TO_CREATE_TEMPLATE + }); + + const admin = new DbAdmin(templateConfig); + admin.cleanupTemplate(TEMPLATE_NAME); + admin.createSeededTemplate(TEMPLATE_NAME, { + seed: (db, dbName) => { + db.loadSql(sql('test.sql'), dbName); + db.loadSql(sql('roles.sql'), dbName); + } + }); +} + +beforeAll(() => { + totalStart = Date.now(); + setupTemplateDatabase(); +}); + +beforeEach(async () => { + ({ db, pg, teardown } = await getConnections({}, { + template: TEMPLATE_NAME + })); + usedDbNames.push(db?.config?.database ?? '(unknown)'); + start = Date.now(); +}); + +afterEach(async () => { + await teardown(); + const elapsed = Date.now() - start; + const name = expect.getState().currentTestName ?? 'unknown'; + testResults.push({ name, time: elapsed }); +}); + +afterAll(() => { + const totalTime = Date.now() - totalStart; + const uniqueNames = new Set(usedDbNames); + const avg = testResults.reduce((sum, r) => sum + r.time, 0) / testResults.length; + + const summaryLines = [ + `πŸ§ͺ Template DB Benchmark`, + `πŸ“¦ Total DBs Created: ${usedDbNames.length}`, + `πŸ“‚ Template Used: ${TEMPLATE_NAME}`, + `βœ… Unique DBs: ${uniqueNames.size === usedDbNames.length}`, + `⏱️ Test Timings:`, + ...testResults.map(({ name, time }) => ` β€’ ${name}: ${time}ms`), + `🏁 Average Test Time: ${avg.toFixed(2)}ms`, + `πŸ•’ Total Test Time: ${totalTime}ms` + ]; + + console.log('\n' + summaryLines.join('\n') + '\n'); +}); + +describe('Template DB Benchmark', () => { + it('inserts Alice', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('alice')`); + const res = await pg.query(`SELECT COUNT(*) FROM app_public.users`); + expect(res.rows[0].count).toBe('1'); + }); + + it('starts clean without Alice', async () => { + const res = await pg.query(`SELECT * FROM app_public.users WHERE username = 'alice'`); + expect(res.rows).toHaveLength(0); + }); + + it('inserts Bob, settings should be empty', async () => { + await pg.query(`INSERT INTO app_public.users (username) VALUES ('bob')`); + const settings = await pg.query(`SELECT * FROM app_public.user_settings`); + expect(settings.rows).toHaveLength(0); + }); +}); diff --git a/packages/pgsql-test/jest.config.js b/packages/pgsql-test/jest.config.js new file mode 100644 index 0000000000..0aa3aaa499 --- /dev/null +++ b/packages/pgsql-test/jest.config.js @@ -0,0 +1,18 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest", + testEnvironment: "node", + transform: { + "^.+\\.tsx?$": [ + "ts-jest", + { + babelConfig: false, + tsconfig: "tsconfig.json", + }, + ], + }, + transformIgnorePatterns: [`/node_modules/*`], + testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", + moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"], + modulePathIgnorePatterns: ["dist/*"] +}; diff --git a/packages/pgsql-test/package.json b/packages/pgsql-test/package.json new file mode 100644 index 0000000000..6dadd2abb0 --- /dev/null +++ b/packages/pgsql-test/package.json @@ -0,0 +1,39 @@ +{ + "name": "pgsql-test", + "version": "0.0.1", + "author": "Dan Lynch ", + "description": "PostgreSQL Testing in TypeScript", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", + "homepage": "https://github.com/launchql/launchql", + "license": "SEE LICENSE IN LICENSE", + "publishConfig": { + "access": "public", + "directory": "dist" + }, + "repository": { + "type": "git", + "url": "https://github.com/launchql/launchql" + }, + "bugs": { + "url": "https://github.com/launchql/launchql/issues" + }, + "scripts": { + "copy": "copyfiles -f ../../LICENSE README.md package.json dist", + "clean": "rimraf dist/**", + "prepare": "npm run build", + "build": "npm run clean; tsc; tsc -p tsconfig.esm.json; npm run copy", + "build:dev": "npm run clean; tsc --declarationMap; tsc -p tsconfig.esm.json; npm run copy", + "lint": "eslint . --fix", + "test": "jest", + "test:watch": "jest --watch" + }, + "dependencies": { + "@launchql/types": "^2.0.4", + "@launchql/server-utils": "^2.0.4", + "@launchql/migrate": "^2.0.11", + "chalk": "^4.1.0", + "deepmerge": "^4.3.1" + } +} \ No newline at end of file diff --git a/packages/pgsql-test/sql/roles.sql b/packages/pgsql-test/sql/roles.sql new file mode 100644 index 0000000000..1983433bc3 --- /dev/null +++ b/packages/pgsql-test/sql/roles.sql @@ -0,0 +1,48 @@ +BEGIN; +DO $do$ +BEGIN + IF NOT EXISTS ( + SELECT + FROM + pg_catalog.pg_roles + WHERE + rolname = 'administrator') THEN + CREATE ROLE administrator; +END IF; + IF NOT EXISTS ( + SELECT + FROM + pg_catalog.pg_roles + WHERE + rolname = 'anonymous') THEN + CREATE ROLE anonymous; +END IF; + IF NOT EXISTS ( + SELECT + FROM + pg_catalog.pg_roles + WHERE + rolname = 'authenticated') THEN + CREATE ROLE authenticated; +END IF; +END +$do$; +ALTER USER administrator WITH NOCREATEDB; +ALTER USER administrator WITH NOCREATEROLE; +ALTER USER administrator WITH NOLOGIN; +ALTER USER administrator WITH NOREPLICATION; +ALTER USER administrator WITH BYPASSRLS; +ALTER USER anonymous WITH NOCREATEDB; +ALTER USER anonymous WITH NOCREATEROLE; +ALTER USER anonymous WITH NOLOGIN; +ALTER USER anonymous WITH NOREPLICATION; +ALTER USER anonymous WITH NOBYPASSRLS; +ALTER USER authenticated WITH NOCREATEDB; +ALTER USER authenticated WITH NOCREATEROLE; +ALTER USER authenticated WITH NOLOGIN; +ALTER USER authenticated WITH NOREPLICATION; +ALTER USER authenticated WITH NOBYPASSRLS; +GRANT anonymous TO administrator; +GRANT authenticated TO administrator; +COMMIT; + diff --git a/packages/pgsql-test/sql/test.sql b/packages/pgsql-test/sql/test.sql new file mode 100644 index 0000000000..ea7f70dd7a --- /dev/null +++ b/packages/pgsql-test/sql/test.sql @@ -0,0 +1,36 @@ +-- https://en.wikipedia.org/wiki/Role-based_access_control +BEGIN; +CREATE EXTENSION IF NOT EXISTS citext; +DROP SCHEMA IF EXISTS app_public CASCADE; +CREATE SCHEMA app_public; +CREATE TABLE app_public.users ( + id serial PRIMARY KEY, + username citext, + UNIQUE (username), + CHECK (length(username) < 127) +); +CREATE TABLE app_public.roles ( + id serial PRIMARY KEY, + org_id bigint NOT NULL REFERENCES app_public.users (id) +); +CREATE TABLE app_public.user_settings ( + user_id bigint NOT NULL PRIMARY KEY REFERENCES app_public.users (id), + setting1 text, + UNIQUE (user_id) +); +CREATE TABLE app_public.permissions ( + id serial PRIMARY KEY, + name citext +); +CREATE TABLE app_public.permission_assignment ( + perm_id bigint NOT NULL REFERENCES app_public.permissions (id), + role_id bigint NOT NULL REFERENCES app_public.roles (id), + PRIMARY KEY (perm_id, role_id) +); +CREATE TABLE app_public.subject_assignment ( + subj_id bigint NOT NULL REFERENCES app_public.users (id), + role_id bigint NOT NULL REFERENCES app_public.roles (id), + PRIMARY KEY (subj_id, role_id) +); +COMMIT; + diff --git a/packages/pgsql-test/src/admin.ts b/packages/pgsql-test/src/admin.ts new file mode 100644 index 0000000000..f593297113 --- /dev/null +++ b/packages/pgsql-test/src/admin.ts @@ -0,0 +1,172 @@ +import { execSync } from 'child_process'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; +import { existsSync } from 'fs'; +import { streamSql as stream } from './stream'; + +export interface SeedAdapter { + seed(db: DbAdmin, dbName: string): Promise | void; +} + +export function sqlFileSeedAdapter(files: string[]): SeedAdapter { + return { + seed(db, dbName) { + for (const file of files) { + db.loadSql(file, dbName); + } + } + }; +} + +export function programmaticSeedAdapter( + fn: (db: DbAdmin, dbName: string) => Promise +): SeedAdapter { + return { + seed: fn + }; +} + +export function compositeSeedAdapter(adapters: SeedAdapter[]): SeedAdapter { + return { + async seed(db, dbName) { + for (const adapter of adapters) { + await adapter.seed(db, dbName); + } + } + }; +} + +export class DbAdmin { + constructor( + private config: PgConfig, + private verbose: boolean = false + ) { + this.config = getPgEnvOptions(config); + } + + private getEnv(): Record { + return { + PGHOST: this.config.host, + PGPORT: String(this.config.port), + PGUSER: this.config.user, + PGPASSWORD: this.config.password, + }; + } + + private run(command: string): void { + execSync(command, { + stdio: this.verbose ? 'inherit' : 'pipe', + env: { + ...process.env, + ...this.getEnv(), + }, + }); + } + + private safeDropDb(name: string): void { + try { + this.run(`dropdb "${name}"`); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (!message.includes('does not exist')) { + console.warn(`⚠️ Could not drop database ${name}: ${message}`); + } + } + } + + drop(dbName?: string): void { + this.safeDropDb(dbName ?? this.config.database); + } + + dropTemplate(dbName: string): void { + this.run(`psql -c "UPDATE pg_database SET datistemplate='false' WHERE datname='${dbName}';"`); + this.drop(dbName); + } + + create(dbName?: string): void { + const db = dbName ?? this.config.database; + this.run(`createdb -U ${this.config.user} -h ${this.config.host} -p ${this.config.port} "${db}"`); + } + + createFromTemplate(template: string, dbName?: string): void { + const db = dbName ?? this.config.database; + this.run(`createdb -U ${this.config.user} -h ${this.config.host} -p ${this.config.port} -e "${db}" -T "${template}"`); + } + + installExtensions(extensions: string[] | string, dbName?: string): void { + const db = dbName ?? this.config.database; + const extList = typeof extensions === 'string' ? extensions.split(',') : extensions; + + for (const extension of extList) { + this.run(`psql --dbname "${db}" -c 'CREATE EXTENSION IF NOT EXISTS "${extension}" CASCADE;'`); + } + } + + connectionString(dbName?: string): string { + const { user, password, host, port } = this.config; + const db = dbName ?? this.config.database; + return `postgres://${user}:${password}@${host}:${port}/${db}`; + } + + createTemplateFromBase(base: string, template: string): void { + this.run(`createdb -T "${base}" "${template}"`); + this.run(`psql -c "UPDATE pg_database SET datistemplate = true WHERE datname = '${template}';"`); + } + + cleanupTemplate(template: string): void { + try { + this.run(`psql -c "UPDATE pg_database SET datistemplate = false WHERE datname = '${template}'"`); + } catch { } + this.safeDropDb(template); + } + + async grantRole(role: string, user: string, dbName?: string): Promise { + const db = dbName ?? this.config.database; + const sql = `GRANT ${role} TO ${user};`; + await this.streamSql(sql, db); + } + + async grantConnect(role: string, dbName?: string): Promise { + const db = dbName ?? this.config.database; + const sql = `GRANT CONNECT ON DATABASE "${db}" TO ${role};`; + await this.streamSql(sql, db); + } + + async createUserRole(user: string, password: string, dbName: string): Promise { + const sql = ` + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${user}') THEN + CREATE ROLE ${user} LOGIN PASSWORD '${password}'; + GRANT anonymous TO ${user}; + GRANT authenticated TO ${user}; + END IF; + END $$; + `.trim(); + + this.streamSql(sql, dbName); + } + + loadSql(file: string, dbName: string): void { + if (!existsSync(file)) { + throw new Error(`Missing SQL file: ${file}`); + } + this.run(`psql -f ${file} ${dbName}`); + } + + async streamSql(sql: string, dbName: string): Promise { + await stream({ + ...this.config, + database: dbName + }, sql); + } + + async createSeededTemplate(templateName: string, adapter: SeedAdapter): Promise { + const seedDb = this.config.database; + this.create(seedDb); + await adapter.seed(this, seedDb); + this.cleanupTemplate(templateName); + this.createTemplateFromBase(seedDb, templateName); + this.drop(seedDb); + } + +} diff --git a/packages/pgsql-test/src/connect.ts b/packages/pgsql-test/src/connect.ts new file mode 100644 index 0000000000..74771ea328 --- /dev/null +++ b/packages/pgsql-test/src/connect.ts @@ -0,0 +1,95 @@ +import { DbAdmin } from './admin'; +import { + getEnvOptions, + getPgEnvOptions, + TestConnectionOptions, + PgConfig, + getConnEnvOptions +} from '@launchql/types'; +import { deploy, deployFast, LaunchQLProject } from '@launchql/migrate'; +import { PgTestConnector } from './manager'; +import { randomUUID } from 'crypto'; + +import { teardownPgPools } from '@launchql/server-utils'; + +let manager: PgTestConnector; + +export const getPgRootAdmin = (connOpts: TestConnectionOptions={}) => { + const opts = getPgEnvOptions({ + database: connOpts.rootDb + }); + const admin = new DbAdmin(opts); + return admin; +} + +export const getConnections = async ( + _pgConfig: Partial = {}, + _opts: TestConnectionOptions = {} +) => { + + const connOpts = getConnEnvOptions(_opts); + const config: PgConfig = getPgEnvOptions({ + database: `${connOpts.prefix}${randomUUID()}`, + ..._pgConfig + }); + + const root = getPgRootAdmin(connOpts); + await root.createUserRole( + connOpts.connection.user, + connOpts.connection.password, + connOpts.rootDb + ); + + const admin = new DbAdmin(config); + const proj = new LaunchQLProject(connOpts.cwd); + if (proj.isInModule()) { + admin.create(config.database); + admin.installExtensions(connOpts.extensions); + const opts = getEnvOptions({ + pg: config + }) + if (connOpts.deployFast) { + await deployFast({ + opts, + name: proj.getModuleName(), + database: config.database, + dir: proj.modulePath, + usePlan: true, + verbose: false + }) + } else { + await deploy(opts, proj.getModuleName(), config.database, proj.modulePath); + } + } else { + // Create the test database + if (process.env.TEST_DB) { + config.database = process.env.TEST_DB; + } else if (connOpts.template) { + admin.createFromTemplate(connOpts.template, config.database); + } else { + admin.create(config.database); + admin.installExtensions(connOpts.extensions); + } + + } + + await admin.grantConnect(connOpts.connection.user, config.database); + + // Main admin client (optional unless needed elsewhere) + manager = PgTestConnector.getInstance(); + const pg = manager.getClient(config); + // App user connection + const db = manager.getClient({ + ...config, + user: connOpts.connection.user, + password: connOpts.connection.password + }); + db.setContext({ role: 'anonymous' }); + + const teardown = async () => { + await teardownPgPools(); + await manager.closeAll(); + }; + + return { pg, db, teardown, manager }; +}; diff --git a/packages/pgsql-test/src/index.ts b/packages/pgsql-test/src/index.ts new file mode 100644 index 0000000000..687d076f53 --- /dev/null +++ b/packages/pgsql-test/src/index.ts @@ -0,0 +1,2 @@ +export * from './legacy-connect'; +export * from './utils'; \ No newline at end of file diff --git a/packages/pgsql-test/src/legacy-connect.ts b/packages/pgsql-test/src/legacy-connect.ts new file mode 100644 index 0000000000..16433575f1 --- /dev/null +++ b/packages/pgsql-test/src/legacy-connect.ts @@ -0,0 +1,34 @@ +import { PgTestClient } from './test-client'; +import { PgTestConnector } from './manager'; +import { randomUUID } from 'crypto'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; + +export function connect(config: PgConfig): PgTestClient { + const manager = PgTestConnector.getInstance(); + return manager.getClient(config); +} + +export function close(client: PgTestClient): void { + client.close(); +} + +const manager = PgTestConnector.getInstance(); + +export const Connection = { + connect(config: Partial): PgTestClient { + const creds = getPgEnvOptions(config); + return manager.getClient(creds); + }, + + close(client: PgTestClient): void { + client.close(); + }, + + closeAll(): Promise { + return manager.closeAll(); + }, + + getManager(): PgTestConnector { + return manager; + } +}; diff --git a/packages/pgsql-test/src/manager.ts b/packages/pgsql-test/src/manager.ts new file mode 100644 index 0000000000..6ef9117045 --- /dev/null +++ b/packages/pgsql-test/src/manager.ts @@ -0,0 +1,143 @@ +import { Pool } from 'pg'; +import chalk from 'chalk'; +import { DbAdmin } from './admin'; +import { getConnEnvOptions, getPgEnvOptions, PgConfig } from '@launchql/types'; +import { PgTestClient } from './test-client'; + +const SYS_EVENTS = ['SIGTERM']; + +const end = (pool: Pool) => { + try { + if ((pool as any).ended || (pool as any).ending) { + console.warn(chalk.yellow('⚠️ pg pool already ended or ending')); + return; + } + pool.end(); + } catch (err) { + console.error(chalk.red('❌ pg pool termination error:'), err); + } +}; + +export class PgTestConnector { + private static instance: PgTestConnector; + + private readonly clients = new Set(); + private readonly pgPools = new Map(); + private readonly seenDbConfigs = new Map(); + + private verbose = false; + + private constructor(verbose = false) { + this.verbose = verbose; + + SYS_EVENTS.forEach((event) => { + process.on(event, () => { + this.log(chalk.magenta(`⏹ Received ${event}, closing all connections...`)); + this.closeAll(); + }); + }); + } + + static getInstance(verbose = false): PgTestConnector { + if (!PgTestConnector.instance) { + PgTestConnector.instance = new PgTestConnector(verbose); + } + return PgTestConnector.instance; + } + + private log(...args: any[]) { + if (this.verbose) console.log(...args); + } + + private poolKey(config: PgConfig): string { + return `${config.user}@${config.host}:${config.port}/${config.database}`; + } + + private dbKey(config: PgConfig): string { + return `${config.host}:${config.port}/${config.database}`; + } + + getPool(config: PgConfig): Pool { + const key = this.poolKey(config); + if (!this.pgPools.has(key)) { + const pool = new Pool(config); + this.pgPools.set(key, pool); + this.log(chalk.blue(`πŸ“˜ Created new pg pool: ${chalk.white(key)}`)); + } + return this.pgPools.get(key)!; + } + + getClient(config: PgConfig): PgTestClient { + const client = new PgTestClient(config); + this.clients.add(client); + + const key = this.dbKey(config); + this.seenDbConfigs.set(key, config); + + this.log(chalk.green(`πŸ”Œ New PgTestClient connected to ${config.database}`)); + return client; + } + + async closeAll(): Promise { + this.log(chalk.cyan('\n🧹 Closing all PgTestClients...')); + await Promise.all( + Array.from(this.clients).map(async (client) => { + try { + await client.close(); + this.log(chalk.green(`βœ… Closed client for ${client.config.database}`)); + } catch (err) { + console.warn(chalk.red(`❌ Error closing PgTestClient for ${client.config.database}:`), err); + } + }) + ); + this.clients.clear(); + + this.log(chalk.cyan('\n🧯 Disposing pg pools...')); + for (const [key, pool] of this.pgPools.entries()) { + this.log(chalk.gray(`🧯 Disposing pg pool [${key}]`)); + end(pool); + } + this.pgPools.clear(); + + this.log(chalk.cyan('\nπŸ—‘οΈ Dropping seen databases...')); + await Promise.all( + Array.from(this.seenDbConfigs.values()).map(async (config) => { + try { + // somehow an "admin" db had app_user creds? + const rootPg = getPgEnvOptions(); + const admin = new DbAdmin( + {...config, user: rootPg.user, password: rootPg.password}, + this.verbose + ); + // console.log(config); + admin.drop(); + this.log(chalk.yellow(`🧨 Dropped database: ${chalk.white(config.database)}`)); + } catch (err) { + console.warn(chalk.red(`❌ Failed to drop database ${config.database}:`), err); + } + }) + ); + this.seenDbConfigs.clear(); + + this.log(chalk.green('\nβœ… All PgTestClients closed, pools disposed, databases dropped.')); + } + + close(): void { + this.closeAll(); + } + + drop(config: PgConfig): void { + const key = this.dbKey(config); + // for drop, no need for conn opts + const admin = new DbAdmin(config, this.verbose); + admin.drop(); + this.log(chalk.red(`🧨 Dropped database: ${chalk.white(config.database)}`)); + this.seenDbConfigs.delete(key); + } + + kill(client: PgTestClient): void { + client.close(); + this.drop(client.config); + } + +} diff --git a/packages/pgsql-test/src/stream.ts b/packages/pgsql-test/src/stream.ts new file mode 100644 index 0000000000..ef29dd0cc8 --- /dev/null +++ b/packages/pgsql-test/src/stream.ts @@ -0,0 +1,53 @@ +import { spawn } from 'child_process'; +import { Readable } from 'stream'; +import { env } from 'process'; +import { getSpawnEnvWithPg, PgConfig } from '@launchql/types'; + +function setArgs(config: PgConfig): string[] { + const args = [ + '-U', config.user, + '-h', config.host, + '-d', config.database + ]; + if (config.port) { + args.push('-p', String(config.port)); + } + return args; +} + +// Converts a string to a readable stream (replaces streamify-string) +function stringToStream(text: string): Readable { + const stream = new Readable({ + read() { + this.push(text); + this.push(null); + } + }); + return stream; +} + +export async function streamSql(config: PgConfig, sql: string): Promise { + const args = setArgs(config); + + return new Promise((resolve, reject) => { + const sqlStream = stringToStream(sql); + + const proc = spawn('psql', args, { + env: getSpawnEnvWithPg(config) + }); + + sqlStream.pipe(proc.stdin); + + proc.on('close', (code) => { + resolve(); + }); + + proc.on('error', (error) => { + reject(error); + }); + + proc.stderr.on('data', (data: Buffer) => { + reject(new Error(data.toString())); + }); + }); +} diff --git a/packages/pgsql-test/src/test-client.ts b/packages/pgsql-test/src/test-client.ts new file mode 100644 index 0000000000..c665b7c279 --- /dev/null +++ b/packages/pgsql-test/src/test-client.ts @@ -0,0 +1,109 @@ +import { Client, QueryResult } from 'pg'; +import { PgConfig } from '@launchql/types'; + +export class PgTestClient { + public config: PgConfig; + private client: Client; + private ctxStmts: string = ''; + private _ended: boolean = false; + + constructor(config: PgConfig) { + this.config = config; + this.client = new Client({ + host: this.config.host, + port: this.config.port, + database: this.config.database, + user: this.config.user, + password: this.config.password + }); + this.client.connect(); + } + + close(): void { + if (!this._ended) { + this._ended = true; + this.client.end(); + } + } + + async begin(): Promise { + await this.client.query('BEGIN;'); + } + + async savepoint(name: string = 'lqlsavepoint'): Promise { + await this.client.query(`SAVEPOINT "${name}";`); + } + + async rollback(name: string = 'lqlsavepoint'): Promise { + await this.client.query(`ROLLBACK TO SAVEPOINT "${name}";`); + } + + async commit(): Promise { + await this.client.query('COMMIT;'); + } + + async beforeEach(): Promise { + await this.begin(); + await this.savepoint(); + } + + async afterEach(): Promise { + await this.rollback(); + await this.commit(); + } + + setContext(ctx: Record): void { + this.ctxStmts = Object.entries(ctx) + .map(([key, val]) => + val === null + ? `SELECT set_config('${key}', NULL, true);` + : `SELECT set_config('${key}', '${val}', true);` + ) + .join('\n'); + } + + async any(query: string, values?: any[]): Promise { + const result = await this.query(query, values); + return result.rows; + } + + async one(query: string, values?: any[]): Promise { + const rows = await this.any(query, values); + if (rows.length !== 1) { + throw new Error('Expected exactly one result'); + } + return rows[0]; + } + + async oneOrNone(query: string, values?: any[]): Promise { + const rows = await this.any(query, values); + return rows[0] || null; + } + + async many(query: string, values?: any[]): Promise { + const rows = await this.any(query, values); + if (rows.length === 0) throw new Error('Expected many rows, got none'); + return rows; + } + + async manyOrNone(query: string, values?: any[]): Promise { + return this.any(query, values); + } + + async none(query: string, values?: any[]): Promise { + await this.query(query, values); + } + + async result(query: string, values?: any[]): Promise { + return this.query(query, values); + } + + async query(query: string, values?: any[]): Promise> { + if (this.ctxStmts) { + await this.client.query(this.ctxStmts); + } + const result = await this.client.query(query, values); + return result; + } + +} diff --git a/packages/pgsql-test/test-utils/index.ts b/packages/pgsql-test/test-utils/index.ts new file mode 100644 index 0000000000..edf10e5f31 --- /dev/null +++ b/packages/pgsql-test/test-utils/index.ts @@ -0,0 +1,2 @@ +import path from 'path'; +import fs from 'fs'; diff --git a/packages/pgsql-test/tsconfig.esm.json b/packages/pgsql-test/tsconfig.esm.json new file mode 100644 index 0000000000..800d7506d3 --- /dev/null +++ b/packages/pgsql-test/tsconfig.esm.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "dist/esm", + "module": "es2022", + "rootDir": "src/", + "declaration": false + } +} diff --git a/packages/pgsql-test/tsconfig.json b/packages/pgsql-test/tsconfig.json new file mode 100644 index 0000000000..1a9d5696cb --- /dev/null +++ b/packages/pgsql-test/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src/" + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules", "**/*.spec.*", "**/*.test.*"] +} diff --git a/packages/server-utils/package.json b/packages/server-utils/package.json index 4adf56bef5..53cd509ff5 100644 --- a/packages/server-utils/package.json +++ b/packages/server-utils/package.json @@ -32,7 +32,6 @@ "dependencies": { "@launchql/types": "^2.0.4", "cors": "^2.8.5", - "envalid": "^8.0.0", "express": "^5.1.0", "lru-cache": "^11.1.0", "pg": "^8.15.6", diff --git a/packages/server-utils/src/env.ts b/packages/server-utils/src/env.ts deleted file mode 100644 index 308e614c32..0000000000 --- a/packages/server-utils/src/env.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { cleanEnv, str, port } from 'envalid'; - -export const env = cleanEnv(process.env, { - PGUSER: str({ default: 'postgres' }), - PGHOST: str({ default: 'localhost' }), - PGPASSWORD: str({ default: 'password' }), - PGPORT: port({ default: 5432 }) -}); diff --git a/packages/server-utils/src/lru.ts b/packages/server-utils/src/lru.ts index 1262085e5a..285fa14707 100644 --- a/packages/server-utils/src/lru.ts +++ b/packages/server-utils/src/lru.ts @@ -1,5 +1,4 @@ import { LRUCache } from 'lru-cache'; - import pg from 'pg'; import { HttpRequestHandler } from 'postgraphile'; @@ -10,19 +9,7 @@ const ONE_YEAR = ONE_DAY * 366; // Kubernetes sends only SIGTERM on pod shutdown const SYS_EVENTS = ['SIGTERM']; -const end = (pool: any) => { - try { - if (pool.ended || pool.ending) { - console.error( - 'Avoid calling end() β€” pool is already ended or ending.' - ); - return; - } - pool.end(); - } catch (e) { - process.stderr.write(String(e)); - } -}; +type PgPoolKey = string; export interface GraphileCache { pgPool: pg.Pool; @@ -30,62 +17,183 @@ export interface GraphileCache { handler: HttpRequestHandler; } +// Wrapper for pg.Pool to track disposal state +class ManagedPgPool { + public isDisposed = false; + private disposePromise: Promise | null = null; + + constructor(public readonly pool: pg.Pool, public readonly key: string) {} + + async dispose(): Promise { + if (this.isDisposed) { + return this.disposePromise; + } + + this.isDisposed = true; + this.disposePromise = (async () => { + try { + if (!this.pool.ended) { + await this.pool.end(); + console.log(`βœ… pg.Pool ${this.key} ended.`); + } else { + console.log(`β˜‘οΈ pg.Pool ${this.key} ALREADY ended.`); + } + } catch (err) { + console.error(`❌ Error ending pg.Pool ${this.key}:`, err); + // Re-throw to ensure Promise.allSettled captures the error + throw err; + } + })(); + + return this.disposePromise; + } +} + +// +// --- Service Cache --- +// +export const svcCache = new LRUCache({ + max: 25, + ttl: ONE_YEAR, + updateAgeOnGet: true, + dispose: (svc, key) => { + console.log(`πŸ—‘οΈ Disposing service[${key}]`); + } +}); + +// // --- Graphile Cache --- +// export const graphileCache = new LRUCache({ max: 15, - dispose: (obj: GraphileCache, key: string) => { - console.log(`disposing PostGraphile[${key}]`); - }, - updateAgeOnGet: true, ttl: ONE_YEAR, + updateAgeOnGet: true, + dispose: (obj, key) => { + console.log(`πŸ—‘οΈ Disposing PostGraphile[${key}]`); + } }); -// --- Postgres Pool Cache --- -export const pgCache = new LRUCache({ - max: 10, - dispose: (pgPool: pg.Pool, key: string) => { - console.log(`disposing pg ${key}`); - graphileCache.forEach((obj: GraphileCache, k: string) => { - if (obj.pgPoolKey === key) { - graphileCache.delete(k); +// +// --- PgPoolCacheManager --- +// +export class PgPoolCacheManager { + private cleanupTasks: Promise[] = []; + private closed = false; + + private readonly pgCache = new LRUCache({ + max: 10, + ttl: ONE_YEAR, + updateAgeOnGet: true, + dispose: (managedPool, key, reason) => { + console.log(`🧹 Disposing pg pool [${key}] (${reason})`); + this.cleanGraphileDependencies(key); + this.disposePool(managedPool); + }, + }); + + constructor(private readonly graphileCache: LRUCache) {} + + get(key: PgPoolKey): pg.Pool | undefined { + const managedPool = this.pgCache.get(key); + return managedPool?.pool; + } + + has(key: PgPoolKey): boolean { + return this.pgCache.has(key); + } + + set(key: PgPoolKey, pool: pg.Pool): void { + if (this.closed) { + throw new Error('Cannot add to cache after it has been closed'); + } + this.pgCache.set(key, new ManagedPgPool(pool, key)); + } + + delete(key: PgPoolKey): void { + const managedPool = this.pgCache.get(key); + const existed = this.pgCache.delete(key); + if (!existed && managedPool) { + this.cleanGraphileDependencies(key); + this.disposePool(managedPool); + } + } + + clear(): void { + const entries = [...this.pgCache.entries()]; + this.pgCache.clear(); + for (const [key, managedPool] of entries) { + this.cleanGraphileDependencies(key); + this.disposePool(managedPool); + } + } + + async close(): Promise { + if (this.closed) return; + + this.closed = true; + this.clear(); + await this.waitForDisposals(); + } + + async waitForDisposals(): Promise { + if (this.cleanupTasks.length === 0) return; + + const tasks = [...this.cleanupTasks]; + // Clear the array before awaiting to avoid potential race conditions + this.cleanupTasks = []; + + await Promise.allSettled(tasks); + } + + private cleanGraphileDependencies(pgPoolKey: string): void { + this.graphileCache.forEach((entry, k) => { + if (entry.pgPoolKey === pgPoolKey) { + console.log(`🧽 Removing graphileCache[${k}] due to pgPool[${pgPoolKey}]`); + this.graphileCache.delete(k); } }); - end(pgPool); - }, - updateAgeOnGet: true, - ttl: ONE_YEAR, -}); + } -// --- Generic Service Cache --- -export const svcCache = new LRUCache({ - max: 25, - dispose: (svc: any, key: any) => { - console.log(`disposing service[${key}]`); - }, - updateAgeOnGet: true, - ttl: ONE_YEAR, -}); + private disposePool(managedPool: ManagedPgPool): void { + if (managedPool.isDisposed) return; + + const task = managedPool.dispose(); + this.cleanupTasks.push(task); + } +} + +// +// --- Instantiate the pgCache manager --- +// +export const pgCache = new PgPoolCacheManager(graphileCache); +// // --- Graceful Shutdown --- -const once = any>(fn: T, context?: any) => { - let result: ReturnType; - return function (...args: Parameters) { - if (fn) { - // @ts-ignore - result = fn.apply(context || this, args); - fn = null!; - } - return result; - }; -}; +// +const closePromise: { promise: Promise | null } = { promise: null }; -const close = once(() => { - console.log('closing server utils...'); - graphileCache.clear(); - pgCache.clear(); - svcCache.clear(); -}); +export const close = async (verbose: boolean = false): Promise => { + if (closePromise.promise) return closePromise.promise; + + closePromise.promise = (async () => { + if (verbose) console.log('πŸ›‘ Closing all server caches...'); + svcCache.clear(); + graphileCache.clear(); + await pgCache.close(); + if (verbose) console.log('βœ… All caches disposed.'); + })(); + + return closePromise.promise; +}; SYS_EVENTS.forEach((event) => { - process.on(event, close); + process.on(event, () => { + console.log(`πŸ“¦ Received ${event}`); + // Don't await - we want to start the shutdown process immediately + close(); + }); }); + +export const teardownPgPools = async (verbose: boolean = false): Promise => { + return close(verbose); +}; \ No newline at end of file diff --git a/packages/server-utils/src/pg.ts b/packages/server-utils/src/pg.ts index 3094a43e27..1987756a68 100644 --- a/packages/server-utils/src/pg.ts +++ b/packages/server-utils/src/pg.ts @@ -1,7 +1,7 @@ import pg from 'pg'; import { pgCache } from './lru'; -import { PostgresOptions } from '@launchql/types'; +import { getPgEnvOptions, PgConfig } from '@launchql/types'; export const getDbString = ( user: string, @@ -12,21 +12,15 @@ export const getDbString = ( ): string => `postgres://${user}:${password}@${host}:${port}/${database}`; -export const getRootPgPool = ({ - user, - password, - host, - port, - database, -}: PostgresOptions): pg.Pool => { +export const getRootPgPool = (pgConfig: Partial): pg.Pool => { + const config = getPgEnvOptions(pgConfig); + const { user, password, host, port, database, } = config; if (pgCache.has(database)) { const cached = pgCache.get(database); if (cached) return cached; } - const connectionString = getDbString(user, password, host, port, database); const pgPool = new pg.Pool({ connectionString }); - pgCache.set(database, pgPool); return pgPool; -}; +}; \ No newline at end of file diff --git a/packages/server/package.json b/packages/server/package.json index 39e5b89ea9..8b5e5b963d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -44,7 +44,6 @@ "@pyramation/postgis": "^0.1.1", "@pyramation/postgraphile-plugin-fulltext-filter": "^2.0.0", "cors": "^2.8.5", - "envalid": "^8.0.0", "express": "^5.1.0", "graphile-build": "^4.14.1", "graphile-i18n": "^0.0.3", diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 20a938628a..f038eda0c0 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -16,10 +16,10 @@ import { flush, flushService } from './middleware/flush'; import requestIp from 'request-ip'; import { Pool, PoolClient } from 'pg'; -import { LaunchQLOptions, getMergedOptions } from '@launchql/types'; +import { LaunchQLOptions, getEnvOptions } from '@launchql/types'; export const LaunchQLServer = (rawOpts: LaunchQLOptions = {}) => { - const app = new Server(getMergedOptions(rawOpts)); + const app = new Server(getEnvOptions(rawOpts)); app.addEventListener(); app.listen(); }; diff --git a/packages/types/src/env.ts b/packages/types/src/env.ts index 62611c1d7f..4d1995ef74 100644 --- a/packages/types/src/env.ts +++ b/packages/types/src/env.ts @@ -1,76 +1,134 @@ -import { getMergedOptions, LaunchQLOptions } from './launchql'; +import deepmerge from 'deepmerge'; +import { launchqlDefaults, LaunchQLOptions, PgConfig, TestConnectionOptions } from './launchql'; const parseEnvNumber = (val?: string): number | undefined => { - const num = Number(val); - return !isNaN(num) ? num : undefined; + const num = Number(val); + return !isNaN(num) ? num : undefined; }; const parseEnvBoolean = (val?: string): boolean | undefined => { - if (val === undefined) return undefined; - return ['true', '1', 'yes'].includes(val.toLowerCase()); + if (val === undefined) return undefined; + return ['true', '1', 'yes'].includes(val.toLowerCase()); }; export const getEnvOptions = (overrides: LaunchQLOptions = {}): LaunchQLOptions => { - const envOpts = getEnvVars(); - return getMergedOptions - ({ - ...envOpts, - ...overrides, - }); + const envOpts = getEnvVars(); + const defaults = deepmerge(launchqlDefaults, envOpts); + const options = deepmerge(defaults, overrides); + // if you need to sanitize... + return options; }; +export const getPgEnvOptions = (overrides: Partial = {}): PgConfig => { + const envOpts = getPgEnvVars(); + const defaults = deepmerge(launchqlDefaults.pg, envOpts); + const options = deepmerge(defaults, overrides); + // if you need to sanitize... + return options; +}; + +export const getConnEnvOptions = (overrides: Partial = {}): TestConnectionOptions => { + const opts = getEnvOptions({ + db: overrides + }); + return opts.db; +}; + +const getEnvVars = (): LaunchQLOptions => { + const { + PGROOTDATABASE, + + PORT, + SERVER_HOST, + SERVER_TRUST_PROXY, + SERVER_ORIGIN, + SERVER_STRICT_AUTH, -export const getEnvVars = (): LaunchQLOptions => { - const { - PORT, - SERVER_HOST, - SERVER_TRUST_PROXY, - SERVER_ORIGIN, - SERVER_STRICT_AUTH, + PGHOST, + PGPORT, + PGUSER, + PGPASSWORD, + PGDATABASE, - PGHOST, - PGPORT, - PGUSER, - PGPASSWORD, - PGDATABASE, + FEATURES_SIMPLE_INFLECTION, + FEATURES_OPPOSITE_BASE_NAMES, + FEATURES_POSTGIS, - FEATURES_SIMPLE_INFLECTION, - FEATURES_OPPOSITE_BASE_NAMES, - FEATURES_POSTGIS, + BUCKET_NAME, + AWS_REGION, + AWS_ACCESS_KEY, + AWS_SECRET_KEY, + MINIO_ENDPOINT, + } = process.env; - BUCKET_NAME, - AWS_REGION, - AWS_ACCESS_KEY, - AWS_SECRET_KEY, - MINIO_ENDPOINT, - } = process.env; + return { + db: { + ...(PGROOTDATABASE && { rootDb: PGROOTDATABASE }), + }, + server: { + ...(PORT && { port: parseEnvNumber(PORT) }), + ...(SERVER_HOST && { host: SERVER_HOST }), + ...(SERVER_TRUST_PROXY && { trustProxy: parseEnvBoolean(SERVER_TRUST_PROXY) }), + ...(SERVER_ORIGIN && { origin: SERVER_ORIGIN }), + ...(SERVER_STRICT_AUTH && { strictAuth: parseEnvBoolean(SERVER_STRICT_AUTH) }), + }, + pg: { + ...(PGHOST && { host: PGHOST }), + ...(PGPORT && { port: parseEnvNumber(PGPORT) }), + ...(PGUSER && { user: PGUSER }), + ...(PGPASSWORD && { password: PGPASSWORD }), + ...(PGDATABASE && { database: PGDATABASE }), + }, + features: { + ...(FEATURES_SIMPLE_INFLECTION && { simpleInflection: parseEnvBoolean(FEATURES_SIMPLE_INFLECTION) }), + ...(FEATURES_OPPOSITE_BASE_NAMES && { oppositeBaseNames: parseEnvBoolean(FEATURES_OPPOSITE_BASE_NAMES) }), + ...(FEATURES_POSTGIS && { postgis: parseEnvBoolean(FEATURES_POSTGIS) }), + }, + cdn: { + ...(BUCKET_NAME && { bucketName: BUCKET_NAME }), + ...(AWS_REGION && { awsRegion: AWS_REGION }), + ...(AWS_ACCESS_KEY && { awsAccessKey: AWS_ACCESS_KEY }), + ...(AWS_SECRET_KEY && { awsSecretKey: AWS_SECRET_KEY }), + ...(MINIO_ENDPOINT && { minioEndpoint: MINIO_ENDPOINT }), + } + }; +}; + +const getPgEnvVars = (): PgConfig => { + const { + PGHOST, + PGPORT, + PGUSER, + PGPASSWORD, + PGDATABASE + } = process.env; - return { - server: { - ...(PORT && { port: parseEnvNumber(PORT) }), - ...(SERVER_HOST && { host: SERVER_HOST }), - ...(SERVER_TRUST_PROXY && { trustProxy: parseEnvBoolean(SERVER_TRUST_PROXY) }), - ...(SERVER_ORIGIN && { origin: SERVER_ORIGIN }), - ...(SERVER_STRICT_AUTH && { strictAuth: parseEnvBoolean(SERVER_STRICT_AUTH) }), - }, - pg: { - ...(PGHOST && { host: PGHOST }), - ...(PGPORT && { port: parseEnvNumber(PGPORT) }), - ...(PGUSER && { user: PGUSER }), - ...(PGPASSWORD && { password: PGPASSWORD }), - ...(PGDATABASE && { database: PGDATABASE }), - }, - features: { - ...(FEATURES_SIMPLE_INFLECTION && { simpleInflection: parseEnvBoolean(FEATURES_SIMPLE_INFLECTION) }), - ...(FEATURES_OPPOSITE_BASE_NAMES && { oppositeBaseNames: parseEnvBoolean(FEATURES_OPPOSITE_BASE_NAMES) }), - ...(FEATURES_POSTGIS && { postgis: parseEnvBoolean(FEATURES_POSTGIS) }), - }, - cdn: { - ...(BUCKET_NAME && { bucketName: BUCKET_NAME }), - ...(AWS_REGION && { awsRegion: AWS_REGION }), - ...(AWS_ACCESS_KEY && { awsAccessKey: AWS_ACCESS_KEY }), - ...(AWS_SECRET_KEY && { awsSecretKey: AWS_SECRET_KEY }), - ...(MINIO_ENDPOINT && { minioEndpoint: MINIO_ENDPOINT }), - } - }; + return { + ...(PGHOST && { host: PGHOST }), + ...(PGPORT && { port: parseEnvNumber(PGPORT) }), + ...(PGUSER && { user: PGUSER }), + ...(PGPASSWORD && { password: PGPASSWORD }), + ...(PGDATABASE && { database: PGDATABASE }), + }; }; + +export function toPgEnvVars(config: Partial): Record { + const opts = deepmerge(launchqlDefaults.pg, config); + return { + ...(opts.host && { PGHOST: opts.host }), + ...(opts.port && { PGPORT: String(opts.port) }), + ...(opts.user && { PGUSER: opts.user }), + ...(opts.password && { PGPASSWORD: opts.password }), + ...(opts.database && { PGDATABASE: opts.database }), + }; +}; + +export function getSpawnEnvWithPg( + config: Partial, + baseEnv: NodeJS.ProcessEnv = process.env +): NodeJS.ProcessEnv { + return { + ...baseEnv, + ...toPgEnvVars(config) + }; +} \ No newline at end of file diff --git a/packages/types/src/launchql.ts b/packages/types/src/launchql.ts index f1c61369ac..7ea27cb872 100644 --- a/packages/types/src/launchql.ts +++ b/packages/types/src/launchql.ts @@ -43,17 +43,31 @@ declare module 'express-serve-static-core' { } } - -export interface PostgresOptions { - host?: string; - port?: number; - user?: string; - password?: string; - database?: string; +export interface PgConfig { + host: string; + port: number; + user: string; + password: string; + database: string; } +export interface TestConnectionOptions { + rootDb?: string; + template?: string; + prefix?: string; + extensions?: string[]; + cwd?: string; + deployFast?: boolean; + connection?: { + user?: string; + password?: string; + role?: string; + } + } + export interface LaunchQLOptions { - pg?: PostgresOptions; + db?: Partial; + pg?: Partial; graphile?: { isPublic?: boolean; schema?: string | string[]; @@ -85,6 +99,18 @@ export interface LaunchQLOptions { export const launchqlDefaults: LaunchQLOptions = { + db: { + rootDb: 'postgres', + prefix: 'db-', + extensions: [], + cwd: process.cwd(), + deployFast: true, + connection: { + user: 'app_user', + password: 'app_password', + role: 'anonymous' + } + }, pg: { host: 'localhost', port: 5432, @@ -118,12 +144,4 @@ export const launchqlDefaults: LaunchQLOptions = { awsAccessKey: 'minioadmin', awsSecretKey: 'minioadmin' } -}; - - - -export const getMergedOptions = (options: LaunchQLOptions): LaunchQLOptions => { - options = deepmerge(launchqlDefaults, options ?? {}); - // if you need to sanitize... - return options; -}; +}; \ No newline at end of file