From a42d131b07f2a2917dd96466b02f6e40f6d4c13c Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 7 Jul 2026 20:16:00 -0700 Subject: [PATCH 01/10] feat(backend): add BullMQ JobManager framework Introduces a JobManager over BullMQ: work-queue Workloads with a ProcessContext, CronWorkloads multiplexed onto a shared cron worker with a reconcile() sweep helper, a JobProducer that owns the queues and the deduplicated enqueue path, and a Redis-backed read model (status/jobDetail). Wired into the backend entrypoint with a demo workload. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/backend/src/index.ts | 48 ++++ packages/backend/src/jobManager.test.ts | 114 +++++++++ packages/backend/src/jobManager.ts | 309 ++++++++++++++++++++++++ packages/backend/src/jobProducer.ts | 32 +++ packages/backend/src/types.ts | 86 ++++++- 5 files changed, 588 insertions(+), 1 deletion(-) create mode 100644 packages/backend/src/jobManager.test.ts create mode 100644 packages/backend/src/jobManager.ts create mode 100644 packages/backend/src/jobProducer.ts diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index b97fe248f..379ca388c 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -20,6 +20,8 @@ import { shutdownPosthog } from "./posthog.js"; import { PromClient } from './promClient.js'; import { RepoIndexManager } from "./repoIndexManager.js"; import { redis } from "./redis.js"; +import { BullMQJobManager, reconcile } from "./jobManager.js"; +import { QueueSpec, Workload } from "./types.js"; const logger = createLogger('backend-entrypoint'); @@ -83,6 +85,51 @@ const api = new Api( logger.info('Worker started.'); +// Background jobs run through the JobManager (BullMQ/Redis as the source of truth). Phase 0 +// wires the framework here in place of the old per-manager pollers; the real workloads +// (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. +const jobManager = new BullMQJobManager(redis); + + +const demoSpec: QueueSpec<{ id: string }> = { + name: 'demo', + dedupKey: ({ id }) => `demo:${id}`, + jobOptions: { + attempts: 2, + backoff: { type: 'fixed', delayMs: 1000 }, + keep: { completed: 50, failed: 50 }, + }, +}; + +const demoWorkload: Workload<{ id: string }, { id: string; ranAt: number }> = { + spec: demoSpec, + concurrency: 2, + process: async ({ data: { id }, jobId, attemptsMade }) => { + logger.info(`demo: processing "${id}" (job ${jobId}, attempt ${attemptsMade})`); + await new Promise((resolve) => setTimeout(resolve, 1500)); + if (id === 'gamma') { + throw new Error(`demo: simulated failure processing "${id}"`); + } + return { id, ranAt: Date.now() }; + }, + onTerminalFailure: async ({ id }, err) => { + logger.warn(`demo: "${id}" failed terminally: ${err.message}`); + }, +}; +jobManager.register(demoWorkload); + +// The reconcile sweep for the demo queue, expressed as a cron workload: every 15s it triggers +// a fixed set into `demo`. Dedup keeps an in-flight item from re-queuing, but a finished one +// re-enqueues on the next tick, so the loop visibly cycles. +jobManager.registerCron(reconcile({ + name: 'demo-sweep', + schedule: { every: '15s' }, + target: 'demo', + scan: async () => ['alpha', 'beta', 'gamma'].map((id) => ({ id })), +})); + +await jobManager.start(); + const listenToShutdownSignals = () => { const signals = SHUTDOWN_SIGNALS; @@ -104,6 +151,7 @@ const listenToShutdownSignals = () => { await auditLogPruner.dispose() await attachmentPruner.dispose() await configManager.dispose() + await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts new file mode 100644 index 000000000..c65b1c06d --- /dev/null +++ b/packages/backend/src/jobManager.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test, vi } from 'vitest'; + +// The module under test creates a logger at import time; stub it so importing pure helpers +// has no side effects (mirrors repoIndexManager.test.ts). +vi.mock('@sourcebot/shared', () => ({ + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), +})); + +// Mock the constants module directly so its env-derived cache-dir paths don't load. +vi.mock('./constants.js', () => ({ + WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, +})); + +import { normalizeJobState, parseDuration, reconcile } from './jobManager.js'; + +describe('parseDuration', () => { + test.each([ + ['500ms', 500], + ['30s', 30_000], + ['5m', 300_000], + ['6h', 21_600_000], + ['1d', 86_400_000], + ])('parses %s', (input, expected) => { + expect(parseDuration(input)).toBe(expected); + }); + + test('trims surrounding whitespace', () => { + expect(parseDuration(' 10m ')).toBe(600_000); + }); + + test.each(['', '5', 'm', '5x', '1.5h', '-5m', '5 m'])('throws on malformed "%s"', (input) => { + expect(() => parseDuration(input)).toThrow(); + }); +}); + +describe('normalizeJobState', () => { + test.each([ + 'waiting', + 'active', + 'delayed', + 'completed', + 'failed', + 'paused', + ])('passes through "%s"', (state) => { + expect(normalizeJobState(state)).toBe(state); + }); + + test('collapses prioritized and waiting-children to waiting', () => { + expect(normalizeJobState('prioritized')).toBe('waiting'); + expect(normalizeJobState('waiting-children')).toBe('waiting'); + }); + + test('maps anything unrecognized to unknown', () => { + expect(normalizeJobState('something-else')).toBe('unknown'); + expect(normalizeJobState('unknown')).toBe('unknown'); + }); +}); + +describe('reconcile', () => { + test('produces a cron workload carrying the given name and schedule', () => { + const cron = reconcile({ + name: 'x-sweep', + schedule: { every: '5m' }, + target: 'x', + scan: async () => [], + }); + expect(cron.name).toBe('x-sweep'); + expect(cron.schedule).toEqual({ every: '5m' }); + }); + + test('handler triggers each scanned item into the target, in order', async () => { + const triggered: Array<{ workload: string; data: unknown }> = []; + const cron = reconcile({ + name: 'x-sweep', + schedule: { pattern: '*/5 * * * *' }, + target: 'x', + scan: async () => [{ id: 'a' }, { id: 'b' }], + }); + + await cron.handler({ + trigger: async (workload, data) => { + triggered.push({ workload, data }); + }, + }); + + expect(triggered).toEqual([ + { workload: 'x', data: { id: 'a' } }, + { workload: 'x', data: { id: 'b' } }, + ]); + }); + + test('handler triggers nothing when scan returns empty', async () => { + let calls = 0; + const cron = reconcile({ + name: 'empty-sweep', + schedule: { every: '1m' }, + target: 'x', + scan: async () => [], + }); + + await cron.handler({ + trigger: async () => { + calls += 1; + }, + }); + + expect(calls).toBe(0); + }); +}); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts new file mode 100644 index 000000000..cd7689ef5 --- /dev/null +++ b/packages/backend/src/jobManager.ts @@ -0,0 +1,309 @@ +import * as Sentry from "@sentry/node"; +import { createLogger } from "@sourcebot/shared"; +import { Job, Queue, Worker } from "bullmq"; +import { Redis } from "ioredis"; +import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; +import { JobProducer } from "./jobProducer.js"; +import { CronWorkload, JobDetail, JobManager, QueueCounts, Schedule, Workload } from "./types.js"; + +const LOG_TAG = 'job-manager'; +const logger = createLogger(LOG_TAG); + +const CRON_QUEUE_NAME = 'cron'; +const CRON_KEEP_COMPLETED = 50; +const CRON_KEEP_FAILED = 200; + +const DURATION_UNITS_MS: Record = { + ms: 1, + s: 1000, + m: 1000 * 60, + h: 1000 * 60 * 60, + d: 1000 * 60 * 60 * 24, +}; + +export const parseDuration = (value: string): number => { + const match = /^(\d+)(ms|s|m|h|d)$/.exec(value.trim()); + if (!match) { + throw new Error(`Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`); + } + return Number(match[1]) * DURATION_UNITS_MS[match[2]]; +}; + +export const normalizeJobState = (state: string): JobDetail['state'] => { + switch (state) { + case 'waiting': + case 'active': + case 'delayed': + case 'completed': + case 'failed': + case 'paused': + return state; + case 'prioritized': + case 'waiting-children': + return 'waiting'; + default: + return 'unknown'; + } +}; + +const scheduleToRepeat = (schedule: Schedule) => + 'pattern' in schedule ? { pattern: schedule.pattern } : { every: parseDuration(schedule.every) }; + +export function reconcile(opts: { + name: string; + schedule: Schedule; + target: string; + scan: () => Promise; +}): CronWorkload { + return { + name: opts.name, + schedule: opts.schedule, + handler: async ({ trigger }) => { + const items = await opts.scan(); + for (const item of items) { + await trigger(opts.target, item); + } + }, + }; +} + +export class BullMQJobManager implements JobManager { + private readonly workloads = new Map>(); + private readonly cronWorkloads = new Map(); + private readonly workers = new Map(); + private readonly producer: JobProducer; + private cronQueue?: Queue; + private cronWorker?: Worker; + private readonly abortController = new AbortController(); + + constructor(private readonly connection: Redis) { + this.producer = new JobProducer(connection); + } + + register(workload: Workload): void { + const name = workload.spec.name; + if (this.workloads.has(name)) { + throw new Error(`Workload "${name}" is already registered`); + } + this.workloads.set(name, workload as unknown as Workload); + } + + registerCron(cron: CronWorkload): void { + if (this.cronWorkloads.has(cron.name)) { + throw new Error(`Cron workload "${cron.name}" is already registered`); + } + this.cronWorkloads.set(cron.name, cron); + } + + async start(): Promise { + if (this.workloads.size === 0 && this.cronWorkloads.size === 0) { + logger.debug('start() called with nothing registered; nothing to do'); + return; + } + + for (const workload of this.workloads.values()) { + this.startWorkload(workload); + } + + if (this.cronWorkloads.size > 0) { + this.cronQueue = new Queue(CRON_QUEUE_NAME, { connection: this.connection }); + this.cronWorker = new Worker( + CRON_QUEUE_NAME, + (job) => this.runCron(job.name), + { connection: this.connection, concurrency: 1 }, + ); + this.cronWorker.on('failed', (job, error) => { + logger.error(`Cron "${job?.name}" run failed: ${error.message}`); + Sentry.captureException(error); + }); + this.cronWorker.on('error', (error) => { + logger.error('Cron worker error:', error); + }); + + for (const cron of this.cronWorkloads.values()) { + await this.cronQueue.upsertJobScheduler( + `cron:${cron.name}`, + scheduleToRepeat(cron.schedule), + { + name: cron.name, + opts: { + removeOnComplete: { count: CRON_KEEP_COMPLETED }, + removeOnFail: { count: CRON_KEEP_FAILED }, + }, + }, + ); + } + } + + logger.info( + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ') || '—'}] ` + + `and ${this.cronWorkloads.size} cron workload(s) [${[...this.cronWorkloads.keys()].join(', ') || '—'}]`, + ); + } + + async trigger(workloadName: string, data: T): Promise { + const workload = this.workloads.get(workloadName); + if (!workload) { + throw new Error(`Cannot trigger unknown workload "${workloadName}"`); + } + await this.producer.enqueue(workload.spec, data); + } + + async status(workloadName: string): Promise { + this.requireRegistered(workloadName); + const counts = await this.producer.queue(workloadName).getJobCounts( + 'waiting', + 'active', + 'delayed', + 'completed', + 'failed', + 'paused', + 'prioritized', + 'waiting-children', + ); + return { + waiting: counts.waiting ?? 0, + active: counts.active ?? 0, + delayed: counts.delayed ?? 0, + completed: counts.completed ?? 0, + failed: counts.failed ?? 0, + paused: counts.paused ?? 0, + prioritized: counts.prioritized ?? 0, + 'waiting-children': counts['waiting-children'] ?? 0, + }; + } + + async jobDetail(workloadName: string, jobId: string): Promise { + this.requireRegistered(workloadName); + const queue = this.producer.queue(workloadName); + const job = await queue.getJob(jobId); + if (!job) { + return null; + } + + const [state, jobLogs] = await Promise.all([ + job.getState(), + queue.getJobLogs(jobId), + ]); + + const enqueuedAt = job.timestamp; + const startedAt = job.processedOn ?? null; + const finishedAt = job.finishedOn ?? null; + + return { + id: job.id ?? jobId, + name: job.name, + state: normalizeJobState(state), + data: job.data, + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + result: job.returnvalue ?? null, + failedReason: job.failedReason ?? null, + stacktrace: job.stacktrace ?? [], + logs: jobLogs.logs, + enqueuedAt, + startedAt, + finishedAt, + waitMs: startedAt !== null ? startedAt - enqueuedAt : null, + runMs: startedAt !== null && finishedAt !== null ? finishedAt - startedAt : null, + }; + } + + async stop(): Promise { + this.abortController.abort(); + + const workers = [...this.workers.values()]; + if (this.cronWorker) { + workers.push(this.cronWorker); + } + await Promise.all(workers.map((worker) => + Promise.race([ + worker.close(), + new Promise((resolve) => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)), + ]), + )); + + if (this.cronQueue) { + await this.cronQueue.close(); + } + await this.producer.close(); + + logger.info('Job manager stopped'); + } + + private startWorkload(workload: Workload): void { + const { spec, concurrency, rateLimit } = workload; + + this.producer.queue(spec.name); + + const worker = new Worker( + spec.name, + (job) => workload.process({ + data: job.data, + jobId: job.id ?? '', + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + signal: this.abortController.signal, + log: async (message) => { await job.log(message); }, + updateProgress: (progress) => job.updateProgress(progress), + }), + { + connection: this.connection, + concurrency, + maxStalledCount: 1, + ...(rateLimit + ? { limiter: { max: rateLimit.max, duration: parseDuration(rateLimit.per) } } + : {}), + }, + ); + + worker.on('failed', (job, error) => { + void this.onWorkloadJobFailed(workload, job, error); + }); + worker.on('error', (error) => { + logger.error(`Worker "${spec.name}" error:`, error); + }); + + this.workers.set(spec.name, worker); + } + + private async runCron(cronName: string): Promise { + const cron = this.cronWorkloads.get(cronName); + if (!cron) { + logger.warn(`Cron fired for unknown workload "${cronName}"; skipping`); + return; + } + await cron.handler({ + trigger: (workload, data) => this.trigger(workload, data), + }); + } + + private async onWorkloadJobFailed( + workload: Workload, + job: Job | undefined, + error: Error, + ): Promise { + if (!job) { + return; + } + const maxAttempts = job.opts.attempts ?? 1; + const isTerminal = job.attemptsMade >= maxAttempts; + if (!isTerminal) { + logger.warn(`Workload "${workload.spec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); + return; + } + logger.error(`Workload "${workload.spec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + try { + await workload.onTerminalFailure?.(job.data, error); + } catch (hookError) { + Sentry.captureException(hookError); + logger.error(`onTerminalFailure for workload "${workload.spec.name}" threw:`, hookError); + } + } + + private requireRegistered(workloadName: string): void { + if (!this.workloads.has(workloadName)) { + throw new Error(`Workload "${workloadName}" is not registered`); + } + } +} diff --git a/packages/backend/src/jobProducer.ts b/packages/backend/src/jobProducer.ts new file mode 100644 index 000000000..f856d0beb --- /dev/null +++ b/packages/backend/src/jobProducer.ts @@ -0,0 +1,32 @@ +import { Queue } from "bullmq"; +import { Redis } from "ioredis"; +import { QueueSpec } from "./types.js"; + +export class JobProducer { + private readonly queues = new Map(); + + constructor(private readonly connection: Redis) {} + + queue(name: string): Queue { + let queue = this.queues.get(name); + if (!queue) { + queue = new Queue(name, { connection: this.connection }); + this.queues.set(name, queue); + } + return queue; + } + + async enqueue(spec: QueueSpec, data: T): Promise { + await this.queue(spec.name).add(spec.name, data, { + deduplication: { id: spec.dedupKey(data) }, + attempts: spec.jobOptions.attempts, + backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + }); + } + + async close(): Promise { + await Promise.all([...this.queues.values()].map((queue) => queue.close())); + } +} diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 8803b48b9..b14d5840b 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -24,4 +24,88 @@ export type RepoAuthCredentials = { * credentials for this repo. */ connectionConfig?: ConnectionConfig; -} \ No newline at end of file +} + +export interface ProcessContext { + data: TData; + jobId: string; + attemptsMade: number; + maxAttempts: number; + signal: AbortSignal; + log(message: string): Promise; + updateProgress(progress: number | object): Promise; +} + +export interface QueueSpec { + name: string; + dedupKey(data: TData): string; + jobOptions: { + attempts: number; + backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + keep: { completed: number; failed: number }; + }; +} + +export interface Workload { + spec: QueueSpec; + concurrency: number; + rateLimit?: { max: number; per: string }; + process(ctx: ProcessContext): Promise; + onTerminalFailure?(data: TData, err: Error): Promise; +} + + +export type Schedule = { every: string } | { pattern: string }; + +export interface JobManager { + register(w: Workload): void; + registerCron(cron: CronWorkload): void; + + start(): Promise; + stop(): Promise; + + trigger(workload: string, data: T): Promise; + + status(workload: string): Promise; + jobDetail(workload: string, jobId: string): Promise; +} + +export interface CronWorkload { + name: string; + schedule: Schedule; + handler(ctx: CronContext): Promise; +} + +export interface CronContext { + trigger(workload: string, data: T): Promise; +} + + +export interface QueueCounts { + waiting: number; + active: number; + delayed: number; + completed: number; + failed: number; + paused: number; + prioritized?: number; + 'waiting-children'?: number; +} + +export interface JobDetail { + id: string; + name: string; + state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; + data: TData; + attemptsMade: number; + maxAttempts: number; + result?: TResult | null; + failedReason?: string | null; + stacktrace?: string[]; + logs: string[]; + enqueuedAt: number; + startedAt: number | null; + finishedAt: number | null; + waitMs?: number | null; + runMs?: number | null; +} From bb84b8e44b3b4d2238333ddc8c1e51618814c410 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 14 Jul 2026 09:55:17 -0700 Subject: [PATCH 02/10] wip --- docs/snippets/schemas/v3/index.schema.mdx | 12 +- packages/backend/src/configManager.ts | 24 +- packages/backend/src/connectionManager.ts | 410 ------------------ packages/backend/src/connectionWorkload.ts | 141 ++++++ .../backend/src/ee/syncSearchContexts.test.ts | 62 ++- packages/backend/src/ee/syncSearchContexts.ts | 19 +- packages/backend/src/index.ts | 163 ++++--- packages/backend/src/jobManager.test.ts | 53 +-- packages/backend/src/jobManager.ts | 131 ++---- packages/backend/src/jobProducer.ts | 10 +- packages/backend/src/types.ts | 147 ++++--- packages/schemas/src/v3/index.schema.ts | 12 +- packages/schemas/src/v3/index.type.ts | 2 + schemas/v3/index.json | 6 +- 14 files changed, 425 insertions(+), 767 deletions(-) delete mode 100644 packages/backend/src/connectionManager.ts create mode 100644 packages/backend/src/connectionWorkload.ts diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 864359251..0a25f6917 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -32,7 +32,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -42,7 +43,8 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -216,7 +218,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -226,7 +229,8 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 4d4f61ff6..74f050fb4 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -1,12 +1,13 @@ -import { Prisma, PrismaClient } from "@sourcebot/db"; +import { Prisma } from "@sourcebot/db"; import { createLogger, env } from "@sourcebot/shared"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { loadConfig } from "@sourcebot/shared"; import chokidar, { FSWatcher } from 'chokidar'; -import { ConnectionManager } from "./connectionManager.js"; import { SINGLE_TENANT_ORG_ID } from "./constants.js"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import isEqual from 'fast-deep-equal'; +import { JobManager } from "./types.js"; +import { prisma } from "./prisma.js"; const logger = createLogger('config-manager'); @@ -14,8 +15,7 @@ export class ConfigManager { private watcher: FSWatcher; constructor( - private db: PrismaClient, - private connectionManager: ConnectionManager, + private jobManager: JobManager, configPath: string, ) { this.watcher = chokidar.watch(configPath, { @@ -46,14 +46,13 @@ export class ConfigManager { await syncSearchContexts({ contexts: config.contexts, orgId: SINGLE_TENANT_ORG_ID, - db: this.db, }); } private syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => { if (connections) { for (const [key, newConnectionConfig] of Object.entries(connections)) { - const existingConnection = await this.db.connection.findUnique({ + const existingConnection = await prisma.connection.findUnique({ where: { name_orgId: { name: key, @@ -73,7 +72,7 @@ export class ConfigManager { // Either update the existing connection or create a new one. const connection = existingConnection ? - await this.db.connection.update({ + await prisma.connection.update({ where: { id: existingConnection.id, }, @@ -84,7 +83,7 @@ export class ConfigManager { enforcePermissionsForPublicRepos, } }) : - await this.db.connection.create({ + await prisma.connection.create({ data: { name: key, config: newConnectionConfig as unknown as Prisma.InputJsonValue, @@ -102,13 +101,16 @@ export class ConfigManager { if (connectionNeedsSyncing) { logger.debug(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`); - await this.connectionManager.createJobs([connection]); + await this.jobManager.trigger('connection', { + connectionId: connection.id, + orgId: SINGLE_TENANT_ORG_ID, + }) } } } // Delete any connections that are no longer in the config. - const deletedConnections = await this.db.connection.findMany({ + const deletedConnections = await prisma.connection.findMany({ where: { isDeclarative: true, name: { @@ -120,7 +122,7 @@ export class ConfigManager { for (const connection of deletedConnections) { logger.debug(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`); - await this.db.connection.delete({ + await prisma.connection.delete({ where: { id: connection.id, } diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts deleted file mode 100644 index c9db057e2..000000000 --- a/packages/backend/src/connectionManager.ts +++ /dev/null @@ -1,410 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { createLogger, env, loadConfig } from "@sourcebot/shared"; -import { Job, Queue, Worker } from "bullmq"; -import { Redis } from 'ioredis'; -import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { syncSearchContexts } from "./ee/syncSearchContexts.js"; -import { captureEvent } from "./posthog.js"; -import { PromClient } from "./promClient.js"; -import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { Settings } from "./types.js"; -import { setIntervalAsync } from "./utils.js"; - -const LOG_TAG = 'connection-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'connection-sync-queue'; - -const CONNECTION_SYNC_TIMEOUT_MS = 1000 * 60 * 60 * 2; // 2 hours - -type JobPayload = { - jobId: string, - connectionId: number, - connectionName: string, - orgId: number, -}; - -type JobResult = { - repoCount: number, -} - -export class ConnectionManager { - private worker: Worker; - private queue: Queue; - private abortController: AbortController; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.worker = new Worker( - QUEUE_NAME, - this.runJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxConnectionSyncJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Connection syncer worker error:`, error); - }); - } - - public startScheduler() { - logger.debug('Starting scheduler'); - this.interval = setIntervalAsync(async () => { - const thresholdDate = new Date(Date.now() - this.settings.resyncConnectionIntervalMs); - const timeoutDate = new Date(Date.now() - CONNECTION_SYNC_TIMEOUT_MS); - - const connections = await this.db.connection.findMany({ - where: { - AND: [ - { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - syncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { status: { in: [ConnectionSyncJobStatus.PENDING, ConnectionSyncJobStatus.IN_PROGRESS] } }, - { createdAt: { gt: timeoutDate } }, - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: ConnectionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - } - ] - } - }); - - if (connections.length > 0) { - await this.createJobs(connections); - } - }, this.settings.resyncConnectionPollingIntervalMs); - } - - - public async createJobs(connections: Connection[]) { - const jobs = await this.db.connectionSyncJob.createManyAndReturn({ - data: connections.map(connection => ({ - connectionId: connection.id, - })), - include: { - connection: true, - } - }); - - for (const job of jobs) { - logger.debug(`Scheduling job ${job.id} for connection ${job.connection.name} (id: ${job.connectionId})`); - await this.queue.add( - 'connection-sync-job', - { - jobId: job.id, - connectionId: job.connectionId, - connectionName: job.connection.name, - orgId: job.connection.orgId, - }, - { jobId: job.id } - ); - - this.promClient.pendingConnectionSyncJobs.inc({ connection: job.connection.name }); - } - - return jobs.map(job => job.id); - } - - private async runJob(job: Job): Promise { - const { jobId, connectionName } = job.data; - const logger = createJobLogger(jobId); - logger.debug(`Running connection sync job ${jobId} for connection ${connectionName} (id: ${job.data.connectionId})`); - - const currentStatus = await this.db.connectionSyncJob.findUniqueOrThrow({ - where: { - id: jobId, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if (currentStatus.status !== ConnectionSyncJobStatus.PENDING && currentStatus.status !== ConnectionSyncJobStatus.IN_PROGRESS) { - throw new Error(`Job ${jobId} is not in a valid state. Expected: ${ConnectionSyncJobStatus.PENDING} or ${ConnectionSyncJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - this.promClient.pendingConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.activeConnectionSyncJobs.inc({ connection: connectionName }); - - const { connection: { config: rawConnectionConfig, orgId } } = await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - }, - select: { - connection: { - select: { - config: true, - orgId: true, - } - } - }, - }); - - const config = rawConnectionConfig as unknown as ConnectionConfig; - - const result = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, job.data.connectionId, this.abortController.signal); - } - case 'gitlab': { - return await compileGitlabConfig(config, job.data.connectionId); - } - case 'gitea': { - return await compileGiteaConfig(config, job.data.connectionId); - } - case 'gerrit': { - return await compileGerritConfig(config, job.data.connectionId); - } - case 'bitbucket': { - return await compileBitbucketConfig(config, job.data.connectionId); - } - case 'azuredevops': { - return await compileAzureDevOpsConfig(config, job.data.connectionId); - } - case 'git': { - return await compileGenericGitHostConfig(config, job.data.connectionId); - } - } - })(); - - let { repoData, warnings } = result; - - await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, - }); - - - // Filter out any duplicates by external_id and external_codeHostUrl. - repoData = repoData.filter((repo, index, self) => { - return index === self.findIndex(r => - r.external_id === repo.external_id && - r.external_codeHostUrl === repo.external_codeHostUrl - ); - }) - - // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, - // and then recreate them when we upsert the repos. For example, if a repo is no-longer - // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't - // appear in the repoData array above, and so the RepoToConnection record won't be re-created. - // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await this.db.$transaction(async (tx) => { - const deleteStart = performance.now(); - await tx.connection.update({ - where: { - id: job.data.connectionId, - }, - data: { - repos: { - deleteMany: {} - } - } - }); - const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`); - - const totalUpsertStart = performance.now(); - for (const repo of repoData) { - const upsertStart = performance.now(); - await tx.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: repo.external_id, - external_codeHostUrl: repo.external_codeHostUrl, - orgId: orgId, - } - }, - update: repo, - create: repo, - }) - const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); - } - const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.debug(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`); - }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); - - return { - repoCount: repoData.length, - }; - } - - - private async onJobCompleted(job: Job, result: JobResult) { - try { - const logger = createJobLogger(job.id!); - const { connectionId, connectionName, orgId } = job.data; - - const { connection } = await this.db.connectionSyncJob.update({ - where: { - id: job.id!, - }, - data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - connection: { - update: { - syncedAt: new Date(), - } - } - }, - select: { - connection: true, - } - }); - - // After a connection has synced, we need to re-sync the org's search contexts as - // there may be new repos that match the search context's include/exclude patterns. - if (env.CONFIG_PATH) { - try { - const config = await loadConfig(env.CONFIG_PATH); - - await syncSearchContexts({ - db: this.db, - orgId, - contexts: config.contexts, - }); - } catch (err) { - logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); - Sentry.captureException(err); - } - } - - logger.debug(`Connection sync job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) completed`); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.connectionSyncJobSuccessTotal.inc({ connection: connectionName }); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_completed', { - connectionId: connectionId, - repoCount: result.repoCount, - type: config.type, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - const jobLogger = createJobLogger(job.id!); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) failed. Retrying...`); - return; - } - - const { connection } = await this.db.connectionSyncJob.update({ - where: { id: job.id }, - data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: job.failedReason, - }, - select: { - connection: true, - } - }); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name }); - this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name }); - - jobLogger.error(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Reason: ${job.failedReason}`); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_failed', { - connectionId: job.data.connectionId, - type: config.type, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - await this.queue.close(); - } -} diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts new file mode 100644 index 000000000..a3b965fb1 --- /dev/null +++ b/packages/backend/src/connectionWorkload.ts @@ -0,0 +1,141 @@ +import { QueueSpec, Workload } from "./types.js"; +import { prisma } from "./prisma.js"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; +import { createLogger, env, loadConfig } from "@sourcebot/shared"; +import { syncSearchContexts } from "./ee/syncSearchContexts.js"; +import * as Sentry from "@sentry/node"; + + +const connectionQueueSpec: QueueSpec<'connection'> = { + name: 'connection', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 } + }, + dedupKey: (data) => `connection:${data.connectionId}` +} + +// @todo +const logger = createLogger('connection-workflow'); + +export const connectionWorkload: Workload<'connection'> = { + spec: connectionQueueSpec, + concurrency: 2, + process: async ({ + data: { + connectionId, + orgId + }, + signal, + }) => { + const connection = await prisma.connection.findUniqueOrThrow({ + where: { + id: connectionId + } + }); + + const config = connection.config as unknown as ConnectionConfig; + + const result = await (async () => { + switch (config.type) { + case 'github': { + return await compileGithubConfig(config, connectionId, signal); + } + case 'gitlab': { + return await compileGitlabConfig(config, connectionId); + } + case 'gitea': { + return await compileGiteaConfig(config, connectionId); + } + case 'gerrit': { + return await compileGerritConfig(config, connectionId); + } + case 'bitbucket': { + return await compileBitbucketConfig(config, connectionId); + } + case 'azuredevops': { + return await compileAzureDevOpsConfig(config, connectionId); + } + case 'git': { + return await compileGenericGitHostConfig(config, connectionId); + } + } + })(); + + let { repoData } = result; + + // Filter out any duplicates by external_id and external_codeHostUrl. + repoData = repoData.filter((repo, index, self) => { + return index === self.findIndex(r => + r.external_id === repo.external_id && + r.external_codeHostUrl === repo.external_codeHostUrl + ); + }) + + // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, + // and then recreate them when we upsert the repos. For example, if a repo is no-longer + // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't + // appear in the repoData array above, and so the RepoToConnection record won't be re-created. + // Repos that have no RepoToConnection records are considered orphaned and can be deleted. + await prisma.$transaction(async (tx) => { + const deleteStart = performance.now(); + await tx.connection.update({ + where: { + id: connectionId, + }, + data: { + repos: { + deleteMany: {} + } + } + }); + const deleteDuration = performance.now() - deleteStart; + logger.debug(`Deleted all RepoToConnection records for connection ${connection.name} (id: ${connectionId}) in ${deleteDuration}ms`); + + const totalUpsertStart = performance.now(); + for (const repo of repoData) { + const upsertStart = performance.now(); + await tx.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, + orgId: orgId, + } + }, + update: repo, + create: repo, + }) + const upsertDuration = performance.now() - upsertStart; + logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); + } + const totalUpsertDuration = performance.now() - totalUpsertStart; + logger.debug(`Upserted ${repoData.length} repos for connection ${connection.name} (id: ${connectionId}) in ${totalUpsertDuration}ms`); + }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); + + await prisma.connection.update({ + where: { + id: connectionId, + }, + data: { + syncedAt: new Date(), + } + }); + + // After a connection has synced, we need to re-sync the org's search contexts as + // there may be new repos that match the search context's include/exclude patterns. + try { + const config = await loadConfig(env.CONFIG_PATH); + + await syncSearchContexts({ + orgId, + contexts: config.contexts, + }); + } catch (err) { + logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); + Sentry.captureException(err); + } + } +} diff --git a/packages/backend/src/ee/syncSearchContexts.test.ts b/packages/backend/src/ee/syncSearchContexts.test.ts index 9aa1decfd..89ffec5d6 100644 --- a/packages/backend/src/ee/syncSearchContexts.test.ts +++ b/packages/backend/src/ee/syncSearchContexts.test.ts @@ -21,6 +21,17 @@ vi.mock('../entitlements.js', () => ({ getPlan: vi.fn(() => Promise.resolve('enterprise')), })); +// `syncSearchContexts` imports the prisma singleton, which builds a real client (and needs +// DATABASE_URL) at import time. Stand in for it with an object that `buildDb` re-populates +// with fresh mocks for each test, so tests stay isolated from one another. +const { prismaMock } = vi.hoisted(() => ({ + prismaMock: {} as Record, +})); + +vi.mock('../prisma.js', () => ({ + prisma: prismaMock, +})); + import { syncSearchContexts } from './syncSearchContexts.js'; // Helper to build a repo record with GitLab topics stored in metadata. @@ -64,20 +75,25 @@ const buildDb = (overrides: Partial<{ connectionFindMany: unknown[]; searchContextFindUnique: unknown; searchContextFindMany: unknown[]; -}> = {}): PrismaClient => ({ - repo: { - findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), - }, - connection: { - findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), - }, - searchContext: { - findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), - findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), - upsert: vi.fn().mockResolvedValue({}), - delete: vi.fn().mockResolvedValue({}), - }, -} as unknown as PrismaClient); +}> = {}): PrismaClient => { + // Overwrite every delegate so no mock survives from a previous test. + Object.assign(prismaMock, { + repo: { + findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), + }, + connection: { + findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), + }, + searchContext: { + findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), + findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), + upsert: vi.fn().mockResolvedValue({}), + delete: vi.fn().mockResolvedValue({}), + }, + }); + + return prismaMock as unknown as PrismaClient; +}; describe('syncSearchContexts - includeTopics', () => { test('includes repos whose topics match an includeTopics entry', async () => { @@ -90,7 +106,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -109,7 +124,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -127,7 +141,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend', 'core'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -146,7 +159,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -165,7 +177,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -188,7 +199,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -209,7 +219,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -230,7 +239,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -251,7 +259,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -274,7 +281,6 @@ describe('syncSearchContexts - includeTopics + excludeTopics combined', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -298,7 +304,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -319,7 +324,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -340,7 +344,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -359,7 +362,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -377,7 +379,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -400,7 +401,6 @@ describe('syncSearchContexts - GitHub excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -422,7 +422,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -446,7 +445,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; diff --git a/packages/backend/src/ee/syncSearchContexts.ts b/packages/backend/src/ee/syncSearchContexts.ts index 6a02c9062..a3592a30f 100644 --- a/packages/backend/src/ee/syncSearchContexts.ts +++ b/packages/backend/src/ee/syncSearchContexts.ts @@ -1,20 +1,19 @@ import micromatch from "micromatch"; import { createLogger } from "@sourcebot/shared"; -import { PrismaClient } from "@sourcebot/db"; import { repoMetadataSchema, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared"; import { hasEntitlement } from "../entitlements.js"; import { SearchContext } from "@sourcebot/schemas/v3/index.type"; +import { prisma } from "../prisma.js"; const logger = createLogger('sync-search-contexts'); interface SyncSearchContextsParams { contexts?: { [key: string]: SearchContext } | undefined; orgId: number; - db: PrismaClient; } export const syncSearchContexts = async (params: SyncSearchContextsParams) => { - const { contexts, orgId, db } = params; + const { contexts, orgId } = params; if (!await hasEntitlement("search-contexts")) { if (contexts) { @@ -25,7 +24,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { if (contexts) { for (const [key, newContextConfig] of Object.entries(contexts)) { - const allRepos = await db.repo.findMany({ + const allRepos = await prisma.repo.findMany({ where: { orgId, }, @@ -44,7 +43,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if(newContextConfig.includeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -101,7 +100,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if (newContextConfig.excludeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -145,7 +144,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { }); } - const currentReposInContext = (await db.searchContext.findUnique({ + const currentReposInContext = (await prisma.searchContext.findUnique({ where: { name_orgId: { name: key, @@ -157,7 +156,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } }))?.repos ?? []; - await db.searchContext.upsert({ + await prisma.searchContext.upsert({ where: { name_orgId: { name: key, @@ -195,7 +194,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } } - const deletedContexts = await db.searchContext.findMany({ + const deletedContexts = await prisma.searchContext.findMany({ where: { name: { notIn: Object.keys(contexts ?? {}), @@ -206,7 +205,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { for (const context of deletedContexts) { logger.debug(`Deleting search context with name '${context.name}'. ID: ${context.id}`); - await db.searchContext.delete({ + await prisma.searchContext.delete({ where: { id: context.id, } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 379ca388c..fee139a87 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -2,26 +2,20 @@ import "./instrument.js"; import * as Sentry from "@sentry/node"; import { createLogger, env, getConfigSettings } from "@sourcebot/shared"; -import { hasEntitlement } from "./entitlements.js"; -import { prisma } from "./prisma.js"; import 'express-async-errors'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; -import { Api } from "./api.js"; -import { AttachmentPruner } from "./attachmentPruner.js"; import { ConfigManager } from "./configManager.js"; -import { ConnectionManager } from './connectionManager.js'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS } from './constants.js'; -import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js"; -import { AuditLogPruner } from "./ee/auditLogPruner.js"; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS, SINGLE_TENANT_ORG_ID } from './constants.js'; import { GithubAppManager } from "./ee/githubAppManager.js"; -import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; +import { hasEntitlement } from "./entitlements.js"; +import { BullMQJobManager } from "./jobManager.js"; import { shutdownPosthog } from "./posthog.js"; +import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from "./repoIndexManager.js"; import { redis } from "./redis.js"; -import { BullMQJobManager, reconcile } from "./jobManager.js"; import { QueueSpec, Workload } from "./types.js"; +import { connectionWorkload } from "./connectionWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -52,36 +46,35 @@ if (await hasEntitlement('github-app')) { await GithubAppManager.getInstance().init(prisma); } -const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); -const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); -const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); -const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); -const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH); -const auditLogPruner = new AuditLogPruner(prisma); -const attachmentPruner = new AttachmentPruner(prisma); - -connectionManager.startScheduler(); -await repoIndexManager.startScheduler(); -auditLogPruner.startScheduler(); -attachmentPruner.startScheduler(); - -if (env.PERMISSION_SYNC_ENABLED === 'true' && !await hasEntitlement('permission-syncing')) { - logger.warn('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); -} -else if (env.PERMISSION_SYNC_ENABLED === 'true' && await hasEntitlement('permission-syncing')) { - if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { - await repoPermissionSyncer.startScheduler(); - } - await accountPermissionSyncer.startScheduler(); -} - -const api = new Api( - promClient, - prisma, - connectionManager, - repoIndexManager, - accountPermissionSyncer, -); +// const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); +// const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); +// const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); +// const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); +// const auditLogPruner = new AuditLogPruner(prisma); +// const attachmentPruner = new AttachmentPruner(prisma); + +// connectionManager.startScheduler(); +// await repoIndexManager.startScheduler(); +// auditLogPruner.startScheduler(); +// attachmentPruner.startScheduler(); + +// if (env.PERMISSION_SYNC_ENABLED === 'true' && !await hasEntitlement('permission-syncing')) { +// logger.warn('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); +// } +// else if (env.PERMISSION_SYNC_ENABLED === 'true' && await hasEntitlement('permission-syncing')) { +// if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { +// await repoPermissionSyncer.startScheduler(); +// } +// await accountPermissionSyncer.startScheduler(); +// } + +// const api = new Api( +// promClient, +// prisma, +// connectionManager, +// repoIndexManager, +// accountPermissionSyncer, +// ); logger.info('Worker started.'); @@ -90,46 +83,50 @@ logger.info('Worker started.'); // (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. const jobManager = new BullMQJobManager(redis); - -const demoSpec: QueueSpec<{ id: string }> = { - name: 'demo', - dedupKey: ({ id }) => `demo:${id}`, +const cronQueueSpec: QueueSpec<'cron'> = { + name: 'cron', jobOptions: { attempts: 2, - backoff: { type: 'fixed', delayMs: 1000 }, - keep: { completed: 50, failed: 50 }, - }, -}; - -const demoWorkload: Workload<{ id: string }, { id: string; ranAt: number }> = { - spec: demoSpec, - concurrency: 2, - process: async ({ data: { id }, jobId, attemptsMade }) => { - logger.info(`demo: processing "${id}" (job ${jobId}, attempt ${attemptsMade})`); - await new Promise((resolve) => setTimeout(resolve, 1500)); - if (id === 'gamma') { - throw new Error(`demo: simulated failure processing "${id}"`); - } - return { id, ranAt: Date.now() }; - }, - onTerminalFailure: async ({ id }, err) => { - logger.warn(`demo: "${id}" failed terminally: ${err.message}`); - }, -}; -jobManager.register(demoWorkload); - -// The reconcile sweep for the demo queue, expressed as a cron workload: every 15s it triggers -// a fixed set into `demo`. Dedup keeps an in-flight item from re-queuing, but a finished one -// re-enqueues on the next tick, so the loop visibly cycles. -jobManager.registerCron(reconcile({ - name: 'demo-sweep', - schedule: { every: '15s' }, - target: 'demo', - scan: async () => ['alpha', 'beta', 'gamma'].map((id) => ({ id })), -})); + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 } + } +} + +const cronWorkload: Workload<'cron'> = { + concurrency: 1, + schedule: { every: '5s' }, + spec: cronQueueSpec, + process: async ({ jobId, trigger }) => { + console.log(`cron ${jobId}`); + + const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connections = await prisma.connection.findMany({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: thresholdDate }} + ] + } + }); + + await Promise.all(connections.map(async (connection) => { + console.log(`Scheduling work for ${connection.id}`); + await trigger('connection', { + connectionId: connection.id, + orgId: SINGLE_TENANT_ORG_ID, + }) + })) + } +} + +jobManager.register(cronWorkload); +jobManager.register(connectionWorkload); await jobManager.start(); +const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); + + const listenToShutdownSignals = () => { const signals = SHUTDOWN_SIGNALS; @@ -144,18 +141,18 @@ const listenToShutdownSignals = () => { logger.info(`Received ${signal}, cleaning up...`); - await repoIndexManager.dispose() - await connectionManager.dispose() - await repoPermissionSyncer.dispose() - await accountPermissionSyncer.dispose() - await auditLogPruner.dispose() - await attachmentPruner.dispose() + // await repoIndexManager.dispose() + // await connectionManager.dispose() + // await repoPermissionSyncer.dispose() + // await accountPermissionSyncer.dispose() + // await auditLogPruner.dispose() + // await attachmentPruner.dispose() await configManager.dispose() await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); - await api.dispose(); + // await api.dispose(); await shutdownPosthog(); logger.info('All workers shut down gracefully'); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index c65b1c06d..5f01538a5 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -16,7 +16,7 @@ vi.mock('./constants.js', () => ({ WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, })); -import { normalizeJobState, parseDuration, reconcile } from './jobManager.js'; +import { normalizeJobState, parseDuration } from './jobManager.js'; describe('parseDuration', () => { test.each([ @@ -61,54 +61,3 @@ describe('normalizeJobState', () => { }); }); -describe('reconcile', () => { - test('produces a cron workload carrying the given name and schedule', () => { - const cron = reconcile({ - name: 'x-sweep', - schedule: { every: '5m' }, - target: 'x', - scan: async () => [], - }); - expect(cron.name).toBe('x-sweep'); - expect(cron.schedule).toEqual({ every: '5m' }); - }); - - test('handler triggers each scanned item into the target, in order', async () => { - const triggered: Array<{ workload: string; data: unknown }> = []; - const cron = reconcile({ - name: 'x-sweep', - schedule: { pattern: '*/5 * * * *' }, - target: 'x', - scan: async () => [{ id: 'a' }, { id: 'b' }], - }); - - await cron.handler({ - trigger: async (workload, data) => { - triggered.push({ workload, data }); - }, - }); - - expect(triggered).toEqual([ - { workload: 'x', data: { id: 'a' } }, - { workload: 'x', data: { id: 'b' } }, - ]); - }); - - test('handler triggers nothing when scan returns empty', async () => { - let calls = 0; - const cron = reconcile({ - name: 'empty-sweep', - schedule: { every: '1m' }, - target: 'x', - scan: async () => [], - }); - - await cron.handler({ - trigger: async () => { - calls += 1; - }, - }); - - expect(calls).toBe(0); - }); -}); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index cd7689ef5..35dfa0af0 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,18 +1,14 @@ import * as Sentry from "@sentry/node"; import { createLogger } from "@sourcebot/shared"; -import { Job, Queue, Worker } from "bullmq"; +import { Job, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; import { JobProducer } from "./jobProducer.js"; -import { CronWorkload, JobDetail, JobManager, QueueCounts, Schedule, Workload } from "./types.js"; +import { DataOf, JobDetail, JobManager, QueueCounts, QueueName, Schedule, Workload } from "./types.js"; const LOG_TAG = 'job-manager'; const logger = createLogger(LOG_TAG); -const CRON_QUEUE_NAME = 'cron'; -const CRON_KEEP_COMPLETED = 50; -const CRON_KEEP_FAILED = 200; - const DURATION_UNITS_MS: Record = { ms: 1, s: 1000, @@ -49,99 +45,43 @@ export const normalizeJobState = (state: string): JobDetail['state'] => { const scheduleToRepeat = (schedule: Schedule) => 'pattern' in schedule ? { pattern: schedule.pattern } : { every: parseDuration(schedule.every) }; -export function reconcile(opts: { - name: string; - schedule: Schedule; - target: string; - scan: () => Promise; -}): CronWorkload { - return { - name: opts.name, - schedule: opts.schedule, - handler: async ({ trigger }) => { - const items = await opts.scan(); - for (const item of items) { - await trigger(opts.target, item); - } - }, - }; -} - export class BullMQJobManager implements JobManager { - private readonly workloads = new Map>(); - private readonly cronWorkloads = new Map(); + private readonly workloads = new Map>(); private readonly workers = new Map(); private readonly producer: JobProducer; - private cronQueue?: Queue; - private cronWorker?: Worker; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { this.producer = new JobProducer(connection); } - register(workload: Workload): void { + register(workload: Workload): void { const name = workload.spec.name; if (this.workloads.has(name)) { throw new Error(`Workload "${name}" is already registered`); } - this.workloads.set(name, workload as unknown as Workload); - } - - registerCron(cron: CronWorkload): void { - if (this.cronWorkloads.has(cron.name)) { - throw new Error(`Cron workload "${cron.name}" is already registered`); - } - this.cronWorkloads.set(cron.name, cron); + this.workloads.set(name, workload); } async start(): Promise { - if (this.workloads.size === 0 && this.cronWorkloads.size === 0) { + if (this.workloads.size === 0) { logger.debug('start() called with nothing registered; nothing to do'); return; } for (const workload of this.workloads.values()) { - this.startWorkload(workload); - } - - if (this.cronWorkloads.size > 0) { - this.cronQueue = new Queue(CRON_QUEUE_NAME, { connection: this.connection }); - this.cronWorker = new Worker( - CRON_QUEUE_NAME, - (job) => this.runCron(job.name), - { connection: this.connection, concurrency: 1 }, - ); - this.cronWorker.on('failed', (job, error) => { - logger.error(`Cron "${job?.name}" run failed: ${error.message}`); - Sentry.captureException(error); - }); - this.cronWorker.on('error', (error) => { - logger.error('Cron worker error:', error); - }); - - for (const cron of this.cronWorkloads.values()) { - await this.cronQueue.upsertJobScheduler( - `cron:${cron.name}`, - scheduleToRepeat(cron.schedule), - { - name: cron.name, - opts: { - removeOnComplete: { count: CRON_KEEP_COMPLETED }, - removeOnFail: { count: CRON_KEEP_FAILED }, - }, - }, - ); - } + await this.startWorkload(workload); } logger.info( - `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ') || '—'}] ` + - `and ${this.cronWorkloads.size} cron workload(s) [${[...this.cronWorkloads.keys()].join(', ') || '—'}]`, + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ')}]`, ); } - async trigger(workloadName: string, data: T): Promise { + async trigger( + workloadName: TName, + data: DataOf + ): Promise { const workload = this.workloads.get(workloadName); if (!workload) { throw new Error(`Cannot trigger unknown workload "${workloadName}"`); @@ -212,29 +152,22 @@ export class BullMQJobManager implements JobManager { async stop(): Promise { this.abortController.abort(); - const workers = [...this.workers.values()]; - if (this.cronWorker) { - workers.push(this.cronWorker); - } - await Promise.all(workers.map((worker) => + await Promise.all([...this.workers.values()].map((worker) => Promise.race([ worker.close(), new Promise((resolve) => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)), ]), )); - if (this.cronQueue) { - await this.cronQueue.close(); - } await this.producer.close(); logger.info('Job manager stopped'); } - private startWorkload(workload: Workload): void { - const { spec, concurrency, rateLimit } = workload; + private async startWorkload(workload: Workload): Promise { + const { spec, concurrency, rateLimit, schedule } = workload; - this.producer.queue(spec.name); + const queue = this.producer.queue(spec.name); const worker = new Worker( spec.name, @@ -246,6 +179,7 @@ export class BullMQJobManager implements JobManager { signal: this.abortController.signal, log: async (message) => { await job.log(message); }, updateProgress: (progress) => job.updateProgress(progress), + trigger: (target, data) => this.trigger(target, data), }), { connection: this.connection, @@ -265,21 +199,30 @@ export class BullMQJobManager implements JobManager { }); this.workers.set(spec.name, worker); - } - private async runCron(cronName: string): Promise { - const cron = this.cronWorkloads.get(cronName); - if (!cron) { - logger.warn(`Cron fired for unknown workload "${cronName}"; skipping`); - return; + if (schedule) { + // @note: jobs produced by BullMQ's scheduler bypass the deduplication check that + // `Queue.add` goes through, so a dedup key would be silently ignored here. A + // scheduled workload gets its overlap protection from `concurrency` instead: the + // next tick's job is only created once the current one goes active, so at most one + // run is ever queued behind the one in flight. + await queue.upsertJobScheduler( + `schedule:${spec.name}`, + scheduleToRepeat(schedule), + { + name: spec.name, + opts: { + attempts: spec.jobOptions.attempts, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + }, + }, + ); } - await cron.handler({ - trigger: (workload, data) => this.trigger(workload, data), - }); } - private async onWorkloadJobFailed( - workload: Workload, + private async onWorkloadJobFailed( + workload: Workload, job: Job | undefined, error: Error, ): Promise { diff --git a/packages/backend/src/jobProducer.ts b/packages/backend/src/jobProducer.ts index f856d0beb..8cd28a2e8 100644 --- a/packages/backend/src/jobProducer.ts +++ b/packages/backend/src/jobProducer.ts @@ -1,6 +1,6 @@ import { Queue } from "bullmq"; import { Redis } from "ioredis"; -import { QueueSpec } from "./types.js"; +import { DataOf, QueueName, QueueSpec } from "./types.js"; export class JobProducer { private readonly queues = new Map(); @@ -16,9 +16,13 @@ export class JobProducer { return queue; } - async enqueue(spec: QueueSpec, data: T): Promise { + async enqueue( + spec: QueueSpec, + data: DataOf + ): Promise { + const dedupKey = spec.dedupKey?.(data); await this.queue(spec.name).add(spec.name, data, { - deduplication: { id: spec.dedupKey(data) }, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), attempts: spec.jobOptions.attempts, backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, removeOnComplete: { count: spec.jobOptions.keep.completed }, diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index b14d5840b..66295017e 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -26,86 +26,109 @@ export type RepoAuthCredentials = { connectionConfig?: ConnectionConfig; } -export interface ProcessContext { - data: TData; - jobId: string; - attemptsMade: number; - maxAttempts: number; - signal: AbortSignal; - log(message: string): Promise; - updateProgress(progress: number | object): Promise; +export interface QueueRegistry { + 'connection': { + connectionId: number, + orgId: number + }, + 'cron': {} } -export interface QueueSpec { - name: string; - dedupKey(data: TData): string; - jobOptions: { - attempts: number; - backoff: { type: 'fixed' | 'exponential'; delayMs: number }; - keep: { completed: number; failed: number }; - }; +export type QueueName = keyof QueueRegistry; +export type DataOf = QueueRegistry[TName]; + +export interface ProcessContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; + signal: AbortSignal; + log(message: string): Promise; + updateProgress(progress: number | object): Promise; + trigger(workload: T, data: DataOf): Promise; } -export interface Workload { - spec: QueueSpec; - concurrency: number; - rateLimit?: { max: number; per: string }; - process(ctx: ProcessContext): Promise; - onTerminalFailure?(data: TData, err: Error): Promise; +/** + * A QueueSpec defines the specification for a queue, including + * it's name, deduplication key, and settings. + */ +export interface QueueSpec { + name: TName; + dedupKey?(data: DataOf): string; + jobOptions: { + attempts: number; + backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + keep: { completed: number; failed: number }; + }; } - export type Schedule = { every: string } | { pattern: string }; -export interface JobManager { - register(w: Workload): void; - registerCron(cron: CronWorkload): void; +/** + * A Workload is a single kind of background work, declared + * as the queue it runs on, the code that processes the job, + * and how much of it may run at once. + * + * Jobs reach a workload's queue in one of two ways: someone calls `trigger`, or - if the + * workload declares a `schedule` - the JobManager enqueues one on that cadence. A sweep is + * just a scheduled workload that carries no payload, and whose `process` scans for work and + * triggers it onto other workloads' queues. + */ +export interface Workload { + spec: QueueSpec; + concurrency: number; + /** + * If set, the JobManager enqueues a job on this cadence rather than waiting for someone to + * `trigger` one. Scheduled jobs carry no payload, so `TData` should be `void`. + */ + schedule?: Schedule; + rateLimit?: { max: number; per: string }; + process(ctx: ProcessContext): Promise; + onTerminalFailure?(data: DataOf, err: Error): Promise; +} - start(): Promise; - stop(): Promise; +export interface JobManager { + register(w: Workload): void; - trigger(workload: string, data: T): Promise; + start(): Promise; + stop(): Promise; - status(workload: string): Promise; - jobDetail(workload: string, jobId: string): Promise; -} + trigger( + workload: TName, + data: DataOf + ): Promise; -export interface CronWorkload { - name: string; - schedule: Schedule; - handler(ctx: CronContext): Promise; + status(workload: string): Promise; + jobDetail(workload: string, jobId: string): Promise; } -export interface CronContext { - trigger(workload: string, data: T): Promise; -} export interface QueueCounts { - waiting: number; - active: number; - delayed: number; - completed: number; - failed: number; - paused: number; - prioritized?: number; - 'waiting-children'?: number; + waiting: number; + active: number; + delayed: number; + completed: number; + failed: number; + paused: number; + prioritized?: number; + 'waiting-children'?: number; } export interface JobDetail { - id: string; - name: string; - state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; - data: TData; - attemptsMade: number; - maxAttempts: number; - result?: TResult | null; - failedReason?: string | null; - stacktrace?: string[]; - logs: string[]; - enqueuedAt: number; - startedAt: number | null; - finishedAt: number | null; - waitMs?: number | null; - runMs?: number | null; + id: string; + name: string; + state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; + data: TData; + attemptsMade: number; + maxAttempts: number; + result?: TResult | null; + failedReason?: string | null; + stacktrace?: string[]; + logs: string[]; + enqueuedAt: number; + startedAt: number | null; + finishedAt: number | null; + waitMs?: number | null; + runMs?: number | null; } diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index 8c1d64b52..c55c72a05 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -31,7 +31,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -41,7 +42,8 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -215,7 +217,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -225,7 +228,8 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 7fa7f5a17..3bede3def 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -100,6 +100,7 @@ export interface Settings { */ resyncConnectionIntervalMs?: number; /** + * @deprecated * The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second. */ resyncConnectionPollingIntervalMs?: number; @@ -108,6 +109,7 @@ export interface Settings { */ reindexRepoPollingIntervalMs?: number; /** + * @deprecated * The number of connection sync jobs to run concurrently. Defaults to 8. */ maxConnectionSyncJobConcurrency?: number; diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 874f9f8d5..6dc6255f1 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -30,7 +30,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -40,7 +41,8 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", From ea54e4cd464ebba9adc5bd886eee4c4463e56208 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 27 Jul 2026 19:10:47 -0700 Subject: [PATCH 03/10] further wip --- packages/backend/src/configManager.ts | 2 +- .../backend/src/connectionWorkload.test.ts | 203 ++++++++++++++++++ packages/backend/src/connectionWorkload.ts | 73 +++++-- packages/backend/src/index.ts | 45 +--- packages/backend/src/jobManager.ts | 135 +++++------- .../backend/src/jobManagerLifecycle.test.ts | 148 +++++++++++++ packages/backend/src/jobProducer.ts | 36 ---- .../src/reconciliationWorkload.test.ts | 97 +++++++++ .../backend/src/reconciliationWorkload.ts | 42 ++++ packages/backend/src/types.ts | 51 +---- packages/shared/package.json | 2 + packages/shared/src/index.server.ts | 13 ++ packages/shared/src/jobProducer.test.ts | 90 ++++++++ packages/shared/src/jobProducer.ts | 66 ++++++ packages/shared/src/queue.ts | 50 +++++ .../src/features/workerApi/actions.test.ts | 132 ++++++++++++ .../web/src/features/workerApi/actions.ts | 39 ++-- packages/web/src/lib/jobProducer.ts | 11 + yarn.lock | 2 + 19 files changed, 1001 insertions(+), 236 deletions(-) create mode 100644 packages/backend/src/connectionWorkload.test.ts create mode 100644 packages/backend/src/jobManagerLifecycle.test.ts delete mode 100644 packages/backend/src/jobProducer.ts create mode 100644 packages/backend/src/reconciliationWorkload.test.ts create mode 100644 packages/backend/src/reconciliationWorkload.ts create mode 100644 packages/shared/src/jobProducer.test.ts create mode 100644 packages/shared/src/jobProducer.ts create mode 100644 packages/shared/src/queue.ts create mode 100644 packages/web/src/features/workerApi/actions.test.ts create mode 100644 packages/web/src/lib/jobProducer.ts diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 74f050fb4..669d3d43e 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -133,4 +133,4 @@ export class ConfigManager { public dispose = async () => { await this.watcher.close(); } -} \ No newline at end of file +} diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts new file mode 100644 index 000000000..8490118fc --- /dev/null +++ b/packages/backend/src/connectionWorkload.test.ts @@ -0,0 +1,203 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + connectionSyncJobCreateMany: vi.fn(), + connectionSyncJobUpdate: vi.fn(), + connectionFindUniqueOrThrow: vi.fn(), + connectionUpdate: vi.fn(), + transactionConnectionUpdate: vi.fn(), + transactionRepoUpsert: vi.fn(), + compileGithubConfig: vi.fn(), + loadConfig: vi.fn(), + syncSearchContexts: vi.fn(), +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('@sourcebot/shared', () => ({ + CONNECTION_QUEUE: { + name: 'connection', + dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + createLogger: vi.fn(() => ({ + debug: vi.fn(), + error: vi.fn(), + })), + env: { + CONFIG_PATH: '/config.json', + CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: 60_000, + }, + loadConfig: mocks.loadConfig, +})); + +vi.mock('./prisma.js', () => ({ + prisma: { + connectionSyncJob: { + createMany: mocks.connectionSyncJobCreateMany, + update: mocks.connectionSyncJobUpdate, + }, + connection: { + findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, + update: mocks.connectionUpdate, + }, + $transaction: vi.fn(async (callback) => callback({ + connection: { + update: mocks.transactionConnectionUpdate, + }, + repo: { + upsert: mocks.transactionRepoUpsert, + }, + })), + }, +})); + +vi.mock('./repoCompileUtils.js', () => ({ + compileAzureDevOpsConfig: vi.fn(), + compileBitbucketConfig: vi.fn(), + compileGenericGitHostConfig: vi.fn(), + compileGerritConfig: vi.fn(), + compileGiteaConfig: vi.fn(), + compileGithubConfig: mocks.compileGithubConfig, + compileGitlabConfig: vi.fn(), +})); + +vi.mock('./ee/syncSearchContexts.js', () => ({ + syncSearchContexts: mocks.syncSearchContexts, +})); + +import { connectionWorkload } from './connectionWorkload.js'; + +const data = { + connectionId: 42, + orgId: 7, +}; + +const lifecycleContext = { + data, + jobId: 'job-1', + attemptsMade: 0, + maxAttempts: 2, +}; + +describe('connectionWorkload lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); + mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); + mocks.connectionSyncJobUpdate.mockResolvedValue({}); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test('creates a pending ConnectionSyncJob when enqueued', async () => { + await connectionWorkload.queueSpec.onEnqueued?.(lifecycleContext); + + expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ + data: [{ + id: 'job-1', + connectionId: 42, + status: 'PENDING', + warningMessages: [], + }], + skipDuplicates: true, + }); + }); + + test('marks the ConnectionSyncJob in progress when processing starts', async () => { + await connectionWorkload.onStarted?.(lifecycleContext); + + expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ + data: [{ + id: 'job-1', + connectionId: 42, + status: 'IN_PROGRESS', + warningMessages: [], + }], + skipDuplicates: true, + }); + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'IN_PROGRESS', + }, + }); + }); + + test('persists compile warnings while processing', async () => { + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: 'github', + config: { + type: 'github', + }, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [], + warnings: ['Repository was archived'], + }); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + + await connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + log: vi.fn(), + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + warningMessages: ['Repository was archived'], + }, + }); + }); + + test('marks the ConnectionSyncJob completed', async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, undefined); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'COMPLETED', + completedAt: new Date('2026-07-27T12:00:00.000Z'), + }, + }); + }); + + test('marks the ConnectionSyncJob failed with its error', async () => { + await connectionWorkload.onTerminalFailure?.( + { ...lifecycleContext, attemptsMade: 2 }, + new Error('Connection failed'), + ); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'FAILED', + completedAt: new Date('2026-07-27T12:00:00.000Z'), + errorMessage: 'Connection failed', + }, + }); + }); +}); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index a3b965fb1..e81b9c954 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -1,33 +1,23 @@ -import { QueueSpec, Workload } from "./types.js"; +import { Workload } from "./types.js"; import { prisma } from "./prisma.js"; +import { ConnectionSyncJobStatus } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { createLogger, env, loadConfig } from "@sourcebot/shared"; +import { CONNECTION_QUEUE, createLogger, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; - -const connectionQueueSpec: QueueSpec<'connection'> = { - name: 'connection', - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 } - }, - dedupKey: (data) => `connection:${data.connectionId}` -} - -// @todo const logger = createLogger('connection-workflow'); export const connectionWorkload: Workload<'connection'> = { - spec: connectionQueueSpec, + queueSpec: CONNECTION_QUEUE, concurrency: 2, process: async ({ data: { connectionId, orgId }, + jobId, signal, }) => { const connection = await prisma.connection.findUniqueOrThrow({ @@ -64,7 +54,16 @@ export const connectionWorkload: Workload<'connection'> = { } })(); - let { repoData } = result; + let { repoData, warnings } = result; + + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + warningMessages: warnings, + }, + }); // Filter out any duplicates by external_id and external_codeHostUrl. repoData = repoData.filter((repo, index, self) => { @@ -137,5 +136,47 @@ export const connectionWorkload: Workload<'connection'> = { logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); Sentry.captureException(err); } + }, + onStarted: async ({ data, jobId }) => { + await prisma.connectionSyncJob.createMany({ + data: [{ + id: jobId, + connectionId: data.connectionId, + status: ConnectionSyncJobStatus.IN_PROGRESS, + warningMessages: [], + }], + skipDuplicates: true, + }); + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId }) => { + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.COMPLETED, + completedAt: new Date(), + }, + }); + }, + onTerminalFailure: async ({ jobId }, error) => { + await prisma.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); } } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index fee139a87..9efa68768 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -6,15 +6,15 @@ import 'express-async-errors'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { ConfigManager } from "./configManager.js"; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS, SINGLE_TENANT_ORG_ID } from './constants.js'; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS } from './constants.js'; import { GithubAppManager } from "./ee/githubAppManager.js"; import { hasEntitlement } from "./entitlements.js"; import { BullMQJobManager } from "./jobManager.js"; import { shutdownPosthog } from "./posthog.js"; import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; +import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; -import { QueueSpec, Workload } from "./types.js"; import { connectionWorkload } from "./connectionWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -83,43 +83,12 @@ logger.info('Worker started.'); // (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. const jobManager = new BullMQJobManager(redis); -const cronQueueSpec: QueueSpec<'cron'> = { - name: 'cron', - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 } - } -} - -const cronWorkload: Workload<'cron'> = { - concurrency: 1, - schedule: { every: '5s' }, - spec: cronQueueSpec, - process: async ({ jobId, trigger }) => { - console.log(`cron ${jobId}`); - - const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); - const connections = await prisma.connection.findMany({ - where: { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate }} - ] - } - }); - - await Promise.all(connections.map(async (connection) => { - console.log(`Scheduling work for ${connection.id}`); - await trigger('connection', { - connectionId: connection.id, - orgId: SINGLE_TENANT_ORG_ID, - }) - })) - } -} +const reconciliationWorkload = createReconciliationWorkload({ + db: prisma, + settings, +}); -jobManager.register(cronWorkload); +jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); await jobManager.start(); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 35dfa0af0..31e935b1f 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,10 +1,9 @@ import * as Sentry from "@sentry/node"; -import { createLogger } from "@sourcebot/shared"; +import { BullMQJobProducer, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; import { Job, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { JobProducer } from "./jobProducer.js"; -import { DataOf, JobDetail, JobManager, QueueCounts, QueueName, Schedule, Workload } from "./types.js"; +import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; const LOG_TAG = 'job-manager'; const logger = createLogger(LOG_TAG); @@ -48,15 +47,15 @@ const scheduleToRepeat = (schedule: Schedule) => export class BullMQJobManager implements JobManager { private readonly workloads = new Map>(); private readonly workers = new Map(); - private readonly producer: JobProducer; + private readonly producer: BullMQJobProducer; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { - this.producer = new JobProducer(connection); + this.producer = new BullMQJobProducer(connection); } register(workload: Workload): void { - const name = workload.spec.name; + const name = workload.queueSpec.name; if (this.workloads.has(name)) { throw new Error(`Workload "${name}" is already registered`); } @@ -81,72 +80,12 @@ export class BullMQJobManager implements JobManager { async trigger( workloadName: TName, data: DataOf - ): Promise { - const workload = this.workloads.get(workloadName); + ): Promise { + const workload = this.workloads.get(workloadName) as Workload | undefined; if (!workload) { throw new Error(`Cannot trigger unknown workload "${workloadName}"`); } - await this.producer.enqueue(workload.spec, data); - } - - async status(workloadName: string): Promise { - this.requireRegistered(workloadName); - const counts = await this.producer.queue(workloadName).getJobCounts( - 'waiting', - 'active', - 'delayed', - 'completed', - 'failed', - 'paused', - 'prioritized', - 'waiting-children', - ); - return { - waiting: counts.waiting ?? 0, - active: counts.active ?? 0, - delayed: counts.delayed ?? 0, - completed: counts.completed ?? 0, - failed: counts.failed ?? 0, - paused: counts.paused ?? 0, - prioritized: counts.prioritized ?? 0, - 'waiting-children': counts['waiting-children'] ?? 0, - }; - } - - async jobDetail(workloadName: string, jobId: string): Promise { - this.requireRegistered(workloadName); - const queue = this.producer.queue(workloadName); - const job = await queue.getJob(jobId); - if (!job) { - return null; - } - - const [state, jobLogs] = await Promise.all([ - job.getState(), - queue.getJobLogs(jobId), - ]); - - const enqueuedAt = job.timestamp; - const startedAt = job.processedOn ?? null; - const finishedAt = job.finishedOn ?? null; - - return { - id: job.id ?? jobId, - name: job.name, - state: normalizeJobState(state), - data: job.data, - attemptsMade: job.attemptsMade, - maxAttempts: job.opts.attempts ?? 1, - result: job.returnvalue ?? null, - failedReason: job.failedReason ?? null, - stacktrace: job.stacktrace ?? [], - logs: jobLogs.logs, - enqueuedAt, - startedAt, - finishedAt, - waitMs: startedAt !== null ? startedAt - enqueuedAt : null, - runMs: startedAt !== null && finishedAt !== null ? finishedAt - startedAt : null, - }; + return this.producer.enqueue(workload.queueSpec, data); } async stop(): Promise { @@ -165,22 +104,24 @@ export class BullMQJobManager implements JobManager { } private async startWorkload(workload: Workload): Promise { - const { spec, concurrency, rateLimit, schedule } = workload; + const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; const queue = this.producer.queue(spec.name); const worker = new Worker( spec.name, - (job) => workload.process({ - data: job.data, - jobId: job.id ?? '', - attemptsMade: job.attemptsMade, - maxAttempts: job.opts.attempts ?? 1, - signal: this.abortController.signal, - log: async (message) => { await job.log(message); }, - updateProgress: (progress) => job.updateProgress(progress), - trigger: (target, data) => this.trigger(target, data), - }), + async (job) => { + const lifecycleContext = this.jobLifecycleContext(job); + await workload.onStarted?.(lifecycleContext); + + return workload.process({ + ...lifecycleContext, + signal: this.abortController.signal, + log: async (message) => { await job.log(message); }, + updateProgress: (progress) => job.updateProgress(progress), + trigger: (target, data) => this.trigger(target, data), + }); + }, { connection: this.connection, concurrency, @@ -194,6 +135,9 @@ export class BullMQJobManager implements JobManager { worker.on('failed', (job, error) => { void this.onWorkloadJobFailed(workload, job, error); }); + worker.on('completed', (job, result) => { + void this.onWorkloadJobCompleted(workload, job, result); + }); worker.on('error', (error) => { logger.error(`Worker "${spec.name}" error:`, error); }); @@ -232,21 +176,38 @@ export class BullMQJobManager implements JobManager { const maxAttempts = job.opts.attempts ?? 1; const isTerminal = job.attemptsMade >= maxAttempts; if (!isTerminal) { - logger.warn(`Workload "${workload.spec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); + logger.warn(`Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); return; } - logger.error(`Workload "${workload.spec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + logger.error(`Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + try { - await workload.onTerminalFailure?.(job.data, error); + await workload.onTerminalFailure?.(this.jobLifecycleContext(job), error); } catch (hookError) { Sentry.captureException(hookError); - logger.error(`onTerminalFailure for workload "${workload.spec.name}" threw:`, hookError); + logger.error(`onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, hookError); } } - private requireRegistered(workloadName: string): void { - if (!this.workloads.has(workloadName)) { - throw new Error(`Workload "${workloadName}" is not registered`); + private async onWorkloadJobCompleted( + workload: Workload, + job: Job, + result: TResult, + ): Promise { + try { + await workload.onCompleted?.(this.jobLifecycleContext(job), result); + } catch (hookError) { + Sentry.captureException(hookError); + logger.error(`onCompleted for workload "${workload.queueSpec.name}" threw:`, hookError); } } + + private jobLifecycleContext(job: Job): JobLifecycleContext { + return { + data: job.data, + jobId: job.id ?? '', + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + }; + } } diff --git a/packages/backend/src/jobManagerLifecycle.test.ts b/packages/backend/src/jobManagerLifecycle.test.ts new file mode 100644 index 000000000..5117233b3 --- /dev/null +++ b/packages/backend/src/jobManagerLifecycle.test.ts @@ -0,0 +1,148 @@ +import { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { Workload } from './types.js'; + +const mocks = vi.hoisted(() => ({ + enqueue: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('@sourcebot/shared', () => ({ + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + BullMQJobProducer: class { + enqueue = mocks.enqueue; + close = mocks.producerClose; + queue = vi.fn(() => ({ + getJobCounts: vi.fn(), + upsertJobScheduler: vi.fn(), + })); + }, +})); + +vi.mock('./constants.js', () => ({ + WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, +})); + +vi.mock('bullmq', () => ({ + Worker: class { + private readonly record: (typeof mocks.workers)[number]; + + constructor(_name: string, processor: (job: unknown) => Promise) { + this.record = { processor, handlers: new Map() }; + mocks.workers.push(this.record); + } + + on(event: string, handler: (...args: unknown[]) => void) { + this.record.handlers.set(event, handler); + } + + close = mocks.workerClose; + }, +})); + +import { BullMQJobManager } from './jobManager.js'; + +const createWorkload = ( + overrides: Partial> = {}, +): Workload<'connection', { repoCount: number }> => ({ + queueSpec: { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + concurrency: 2, + process: vi.fn(async () => ({ repoCount: 3 })), + ...overrides, +}); + +const data = { connectionId: 42, orgId: 1 }; +const job = { + id: 'job-1', + data, + attemptsMade: 2, + opts: { attempts: 2 }, + log: vi.fn(), + updateProgress: vi.fn(), +}; + +describe('BullMQJobManager lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.workers.length = 0; + mocks.enqueue.mockResolvedValue('job-1'); + }); + + test('delegates enqueueing to BullMQJobProducer and returns its job id', async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + const result = await manager.trigger('connection', data); + + expect(result).toBe('job-1'); + expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); + }); + + test('calls onStarted before processing and onCompleted after completion', async () => { + const calls: string[] = []; + const workload = createWorkload({ + onStarted: vi.fn(async () => { calls.push('started'); }), + process: vi.fn(async () => { + calls.push('processed'); + return { repoCount: 3 }; + }), + onCompleted: vi.fn(async () => { calls.push('completed'); }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + expect(calls).toEqual(['started', 'processed']); + + mocks.workers[0].handlers.get('completed')?.(job, result); + await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); + expect(workload.onCompleted).toHaveBeenCalledWith( + expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), + { repoCount: 3 }, + ); + }); + + test('reports lifecycle metadata after terminal failure', async () => { + const onTerminalFailure = vi.fn(); + const workload = createWorkload({ onTerminalFailure }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const error = new Error('failed'); + mocks.workers[0].handlers.get('failed')?.(job, error); + + await vi.waitFor(() => { + expect(onTerminalFailure).toHaveBeenCalledWith({ + data, + jobId: 'job-1', + attemptsMade: 2, + maxAttempts: 2, + }, error); + }); + }); +}); diff --git a/packages/backend/src/jobProducer.ts b/packages/backend/src/jobProducer.ts deleted file mode 100644 index 8cd28a2e8..000000000 --- a/packages/backend/src/jobProducer.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Queue } from "bullmq"; -import { Redis } from "ioredis"; -import { DataOf, QueueName, QueueSpec } from "./types.js"; - -export class JobProducer { - private readonly queues = new Map(); - - constructor(private readonly connection: Redis) {} - - queue(name: string): Queue { - let queue = this.queues.get(name); - if (!queue) { - queue = new Queue(name, { connection: this.connection }); - this.queues.set(name, queue); - } - return queue; - } - - async enqueue( - spec: QueueSpec, - data: DataOf - ): Promise { - const dedupKey = spec.dedupKey?.(data); - await this.queue(spec.name).add(spec.name, data, { - ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), - attempts: spec.jobOptions.attempts, - backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, - removeOnComplete: { count: spec.jobOptions.keep.completed }, - removeOnFail: { count: spec.jobOptions.keep.failed }, - }); - } - - async close(): Promise { - await Promise.all([...this.queues.values()].map((queue) => queue.close())); - } -} diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts new file mode 100644 index 000000000..61db94afb --- /dev/null +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -0,0 +1,97 @@ +import type { PrismaClient } from '@sourcebot/db'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +vi.mock('@sourcebot/shared', () => ({ + RECONCILIATION_QUEUE: { + name: 'reconciliation', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + createLogger: vi.fn(() => ({ + debug: vi.fn(), + })), +})); + +import { createReconciliationWorkload } from './reconciliationWorkload.js'; + +describe('reconciliationWorkload', () => { + const findMany = vi.fn(); + const db = { + connection: { + findMany, + }, + } as unknown as PrismaClient; + + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); + findMany.mockReset(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test('runs every 15 minutes on the reconciliation queue', () => { + const workload = createReconciliationWorkload({ + db, + settings: { + resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, + }, + }); + + expect(workload.queueSpec.name).toBe('reconciliation'); + expect(workload.schedule).toEqual({ every: '15m' }); + expect(workload.concurrency).toBe(1); + }); + + test('triggers connection syncs for connections that are due', async () => { + findMany.mockResolvedValue([ + { id: 42, orgId: 1 }, + { id: 84, orgId: 2 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings: { + resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, + }, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + signal: new AbortController().signal, + log: vi.fn(), + updateProgress: vi.fn(), + trigger, + }); + + expect(findMany).toHaveBeenCalledWith({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + expect(trigger).toHaveBeenCalledTimes(2); + expect(trigger).toHaveBeenCalledWith('connection', { + connectionId: 42, + orgId: 1, + }); + expect(trigger).toHaveBeenCalledWith('connection', { + connectionId: 84, + orgId: 2, + }); + }); +}); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts new file mode 100644 index 000000000..500389b2e --- /dev/null +++ b/packages/backend/src/reconciliationWorkload.ts @@ -0,0 +1,42 @@ +import { PrismaClient } from "@sourcebot/db"; +import { createLogger, RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { Settings, Workload } from "./types.js"; + +const logger = createLogger('reconciliation-workload'); + +interface ReconciliationWorkloadDependencies { + db: PrismaClient; + settings: Pick; +} + +export const createReconciliationWorkload = ({ + db, + settings, +}: ReconciliationWorkloadDependencies): Workload<'reconciliation'> => ({ + concurrency: 1, + schedule: { every: '15m' }, + queueSpec: RECONCILIATION_QUEUE, + process: async ({ trigger }) => { + const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connections = await db.connection.findMany({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: thresholdDate } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + + await Promise.all(connections.map(async (connection) => { + logger.debug(`Scheduling connection sync for connection ${connection.id}`); + await trigger('connection', { + connectionId: connection.id, + orgId: connection.orgId, + }); + })); + }, +}); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 66295017e..ae6b7fc7e 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,6 +1,7 @@ import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; +import { DataOf, JobLifecycleContext, QueueName, QueueSpec } from "@sourcebot/shared"; export type Settings = Required; @@ -14,52 +15,20 @@ export type WithRequired = T & { [P in K]-?: T[P] }; export type RepoWithConnections = Repo & { connections: (RepoToConnection & { connection: Connection })[] }; - export type RepoAuthCredentials = { hostUrl?: string; token: string; cloneUrlWithToken?: string; authHeader?: string; - /** The connection that configured the - * credentials for this repo. - */ connectionConfig?: ConnectionConfig; } -export interface QueueRegistry { - 'connection': { - connectionId: number, - orgId: number - }, - 'cron': {} -} - -export type QueueName = keyof QueueRegistry; -export type DataOf = QueueRegistry[TName]; -export interface ProcessContext { - data: DataOf; - jobId: string; - attemptsMade: number; - maxAttempts: number; +export interface ProcessContext extends JobLifecycleContext { signal: AbortSignal; log(message: string): Promise; updateProgress(progress: number | object): Promise; - trigger(workload: T, data: DataOf): Promise; -} - -/** - * A QueueSpec defines the specification for a queue, including - * it's name, deduplication key, and settings. - */ -export interface QueueSpec { - name: TName; - dedupKey?(data: DataOf): string; - jobOptions: { - attempts: number; - backoff: { type: 'fixed' | 'exponential'; delayMs: number }; - keep: { completed: number; failed: number }; - }; + trigger(workload: T, data: DataOf): Promise; } export type Schedule = { every: string } | { pattern: string }; @@ -75,7 +44,7 @@ export type Schedule = { every: string } | { pattern: string }; * triggers it onto other workloads' queues. */ export interface Workload { - spec: QueueSpec; + queueSpec: QueueSpec; concurrency: number; /** * If set, the JobManager enqueues a job on this cadence rather than waiting for someone to @@ -84,7 +53,12 @@ export interface Workload { schedule?: Schedule; rateLimit?: { max: number; per: string }; process(ctx: ProcessContext): Promise; - onTerminalFailure?(data: DataOf, err: Error): Promise; + /** Called before `process` on every attempt. */ + onStarted?(ctx: JobLifecycleContext): Promise; + /** Called after BullMQ marks the job as completed. */ + onCompleted?(ctx: JobLifecycleContext, result: TResult): Promise; + /** Called after BullMQ exhausts all attempts and marks the job as failed. */ + onTerminalFailure?(ctx: JobLifecycleContext, err: Error): Promise; } export interface JobManager { @@ -96,10 +70,7 @@ export interface JobManager { trigger( workload: TName, data: DataOf - ): Promise; - - status(workload: string): Promise; - jobDetail(workload: string, jobId: string): Promise; + ): Promise; } diff --git a/packages/shared/package.json b/packages/shared/package.json index 6ded4e24b..e865230c3 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -14,10 +14,12 @@ "@google-cloud/secret-manager": "^6.1.1", "@logtail/node": "^0.5.2", "@logtail/winston": "^0.5.2", + "@sentry/node": "^10.40.0", "@sourcebot/db": "workspace:*", "@sourcebot/schemas": "workspace:*", "@t3-oss/env-core": "^0.13.10", "ajv": "^8.17.1", + "bullmq": "^5.34.10", "ioredis": "^5.4.2", "micromatch": "^4.0.8", "strip-json-comments": "^5.0.1", diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 80fe65ce8..753240ab5 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -92,3 +92,16 @@ export { compareVersions, } from "./versionUtils.js"; export type { Version } from "./versionUtils.js"; +export type { + QueueName, + JobLifecycleContext, + DataOf, + QueueSpec, +} from "./queue.js" +export { + CONNECTION_QUEUE, + RECONCILIATION_QUEUE, +} from "./queue.js"; +export { + BullMQJobProducer, +} from "./jobProducer.js"; diff --git a/packages/shared/src/jobProducer.test.ts b/packages/shared/src/jobProducer.test.ts new file mode 100644 index 000000000..ffdad845b --- /dev/null +++ b/packages/shared/src/jobProducer.test.ts @@ -0,0 +1,90 @@ +import type { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import type { QueueSpec } from './queue.js'; + +const queueMocks = vi.hoisted(() => ({ + add: vi.fn(), + close: vi.fn(), +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('./logger.js', () => ({ + createLogger: vi.fn(() => ({ + error: vi.fn(), + })), +})); + +vi.mock('bullmq', () => ({ + Queue: class { + constructor() { + return queueMocks; + } + }, +})); + +import { BullMQJobProducer } from './jobProducer.js'; + +const connectionSpec: QueueSpec<'connection'> = { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, +}; + +const data = { connectionId: 42, orgId: 1 }; + +describe('BullMQJobProducer', () => { + const redis = {} as Redis; + + beforeEach(() => { + vi.clearAllMocks(); + queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); + }); + + test('returns the job id when BullMQ accepts the proposed id', async () => { + const producer = new BullMQJobProducer(redis); + + const result = await producer.enqueue(connectionSpec, data); + + expect(result).toEqual(expect.any(String)); + expect(queueMocks.add).toHaveBeenCalledWith( + 'connection', + data, + expect.objectContaining({ + jobId: result, + deduplication: { id: 'connection:42' }, + }), + ); + }); + + test('calls onEnqueued when a new job is created', async () => { + const onEnqueued = vi.fn(); + const producer = new BullMQJobProducer(redis); + + const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(onEnqueued).toHaveBeenCalledWith({ + data, + jobId: result, + attemptsMade: 0, + maxAttempts: 2, + }); + }); + + test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { + const onEnqueued = vi.fn(); + queueMocks.add.mockResolvedValue({ id: 'existing-job' }); + const producer = new BullMQJobProducer(redis); + + const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(result).toBe('existing-job'); + expect(onEnqueued).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/shared/src/jobProducer.ts b/packages/shared/src/jobProducer.ts new file mode 100644 index 000000000..595a46787 --- /dev/null +++ b/packages/shared/src/jobProducer.ts @@ -0,0 +1,66 @@ +import * as Sentry from "@sentry/node"; +import { Queue } from "bullmq"; +import { randomUUID } from "crypto"; +import { Redis } from "ioredis"; +import { createLogger } from "./logger.js"; +import { DataOf, QueueName, QueueSpec } from "./queue.js"; + +const logger = createLogger('job-producer'); + +export class BullMQJobProducer { + private readonly queues = new Map(); + + constructor(private readonly connection: Redis) {} + + queue(name: string): Queue { + let queue = this.queues.get(name); + if (!queue) { + queue = new Queue(name, { connection: this.connection }); + this.queues.set(name, queue); + } + return queue; + } + + async enqueue( + spec: QueueSpec, + data: DataOf + ): Promise { + const dedupKey = spec.dedupKey?.(data); + const queue = this.queue(spec.name); + + const requestedJobId = randomUUID(); + const job = await queue.add(spec.name, data, { + jobId: requestedJobId, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + attempts: spec.jobOptions.attempts, + backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + }); + + if (!job.id) { + throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); + } + + const isEnqueued = job.id === requestedJobId; + if (isEnqueued && spec.onEnqueued) { + try { + await spec.onEnqueued({ + data, + jobId: job.id, + attemptsMade: 0, + maxAttempts: spec.jobOptions.attempts, + }); + } catch (error) { + Sentry.captureException(error); + logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); + } + } + + return job.id; + } + + async close(): Promise { + await Promise.all([...this.queues.values()].map((queue) => queue.close())); + } +} diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts new file mode 100644 index 000000000..ef9dd67d5 --- /dev/null +++ b/packages/shared/src/queue.ts @@ -0,0 +1,50 @@ + +export type QueueName = keyof QueueRegistry; +export type DataOf = QueueRegistry[TName]; +type EmptyJobData = Record; + +interface QueueRegistry { + 'connection': { + connectionId: number, + orgId: number + }, + 'reconciliation': EmptyJobData, +} + +export const CONNECTION_QUEUE: QueueSpec<'connection'> = { + name: 'connection', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 } + }, + dedupKey: (data) => `connection:${data.connectionId}`, +} + + +export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { + name: 'reconciliation', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, +}; + +export interface QueueSpec { + name: TName; + dedupKey?(data: DataOf): string; + jobOptions: { + attempts: number; + backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + keep: { completed: number; failed: number }; + }; + onEnqueued?(ctx: JobLifecycleContext): Promise; +} + +export interface JobLifecycleContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; +} diff --git a/packages/web/src/features/workerApi/actions.test.ts b/packages/web/src/features/workerApi/actions.test.ts new file mode 100644 index 000000000..0a949d983 --- /dev/null +++ b/packages/web/src/features/workerApi/actions.test.ts @@ -0,0 +1,132 @@ +import { ConnectionSyncJobStatus, OrgRole } from '@sourcebot/db'; +import type { DataOf, QueueSpec } from '@sourcebot/shared'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + connectionFindUnique: vi.fn(), + connectionSyncJobCreateMany: vi.fn(), + enqueue: vi.fn(), +})); + +vi.mock('@/middleware/sew', () => ({ + sew: (fn: () => Promise) => fn(), +})); + +vi.mock('@/middleware/withAuth', () => ({ + withAuth: (fn: (context: unknown) => Promise) => fn({ + org: { id: 7 }, + prisma: { + connection: { + findUnique: mocks.connectionFindUnique, + }, + connectionSyncJob: { + createMany: mocks.connectionSyncJobCreateMany, + }, + }, + role: 'OWNER', + }), + withOptionalAuth: vi.fn(), +})); + +vi.mock('@/middleware/withMinimumOrgRole', () => ({ + withMinimumOrgRole: ( + _role: OrgRole, + _minimumRole: OrgRole, + fn: () => Promise, + ) => fn(), +})); + +vi.mock('@/lib/jobProducer', () => ({ + getJobProducer: () => ({ + enqueue: mocks.enqueue, + }), +})); + +vi.mock('@sourcebot/shared', () => ({ + CONNECTION_QUEUE: { + name: 'connection', + dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + }, + }, + env: { + WORKER_API_URL: 'http://localhost:3060', + }, +})); + +import { syncConnection } from './actions'; + +describe('syncConnection', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); + mocks.enqueue.mockImplementation(async ( + spec: QueueSpec<'connection'>, + data: DataOf<'connection'>, + ) => { + const jobId = 'job-1'; + await spec.onEnqueued?.({ + data, + jobId, + attemptsMade: 0, + maxAttempts: spec.jobOptions.attempts, + }); + return jobId; + }); + }); + + test('enqueues an org-scoped connection sync and creates its pending job record', async () => { + mocks.connectionFindUnique.mockResolvedValue({ + id: 42, + orgId: 7, + }); + + const result = await syncConnection(42); + + expect(mocks.connectionFindUnique).toHaveBeenCalledWith({ + where: { + id: 42, + orgId: 7, + }, + select: { + id: true, + orgId: true, + }, + }); + expect(mocks.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'connection', + onEnqueued: expect.any(Function), + }), + { + connectionId: 42, + orgId: 7, + }, + ); + expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ + data: [{ + id: 'job-1', + connectionId: 42, + status: ConnectionSyncJobStatus.PENDING, + warningMessages: [], + }], + skipDuplicates: true, + }); + expect(result).toEqual({ jobId: 'job-1' }); + }); + + test('does not enqueue a missing connection', async () => { + mocks.connectionFindUnique.mockResolvedValue(null); + + const result = await syncConnection(42); + + expect(mocks.enqueue).not.toHaveBeenCalled(); + expect(result).toEqual(expect.objectContaining({ + statusCode: 404, + message: 'Connection not found', + })); + }); +}); diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index e040ea1e0..b0fa45ac5 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -1,37 +1,40 @@ 'use server'; import { sew } from "@/middleware/sew"; -import { repositoryNotFound, unexpectedError } from "@/lib/serviceError"; +import { notFound, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; -import { OrgRole } from "@sourcebot/db"; -import { env } from "@sourcebot/shared"; +import { ConnectionSyncJobStatus, OrgRole } from "@sourcebot/db"; +import { CONNECTION_QUEUE, env } from "@sourcebot/shared"; import z from "zod"; +import { getJobProducer } from "@/lib/jobProducer"; const WORKER_API_URL = env.WORKER_API_URL; export const syncConnection = async (connectionId: number) => sew(() => - withAuth(({ role }) => + withAuth(({ org, prisma, role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const response = await fetch(`${WORKER_API_URL}/api/sync-connection`, { - method: 'POST', - body: JSON.stringify({ - connectionId - }), - headers: { - 'Content-Type': 'application/json', + const connection = await prisma.connection.findUnique({ + where: { + id: connectionId, + orgId: org.id, + }, + select: { + id: true, + orgId: true, }, }); - if (!response.ok) { - return unexpectedError('Failed to sync connection'); + if (!connection) { + return notFound('Connection not found'); } - const data = await response.json(); - const schema = z.object({ - jobId: z.string(), + const jobId = await getJobProducer().enqueue(CONNECTION_QUEUE, { + connectionId: connection.id, + orgId: connection.orgId, }); - return schema.parse(data); + + return { jobId }; }) ) ); @@ -108,4 +111,4 @@ export const addGithubRepo = async (owner: string, repo: string) => sew(() => }); return schema.parse(data); }) -); \ No newline at end of file +); diff --git a/packages/web/src/lib/jobProducer.ts b/packages/web/src/lib/jobProducer.ts new file mode 100644 index 000000000..1d50ebe6d --- /dev/null +++ b/packages/web/src/lib/jobProducer.ts @@ -0,0 +1,11 @@ +import 'server-only'; + +import { BullMQJobProducer } from '@sourcebot/shared'; +import { getRedisClient } from './redis'; + +let jobProducer: BullMQJobProducer | undefined; + +export function getJobProducer() { + jobProducer ??= new BullMQJobProducer(getRedisClient()); + return jobProducer; +} diff --git a/yarn.lock b/yarn.lock index 088a16ec4..ece7d1c39 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9266,12 +9266,14 @@ __metadata: "@google-cloud/secret-manager": "npm:^6.1.1" "@logtail/node": "npm:^0.5.2" "@logtail/winston": "npm:^0.5.2" + "@sentry/node": "npm:^10.40.0" "@sourcebot/db": "workspace:*" "@sourcebot/schemas": "workspace:*" "@t3-oss/env-core": "npm:^0.13.10" "@types/micromatch": "npm:^4.0.9" "@types/node": "npm:^22.7.5" ajv: "npm:^8.17.1" + bullmq: "npm:^5.34.10" cross-env: "npm:^7.0.3" ioredis: "npm:^5.4.2" micromatch: "npm:^4.0.8" From 809bad44d810c063f2747b9a42b460854736c704 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 28 Jul 2026 15:51:21 -0700 Subject: [PATCH 04/10] further wip --- .../backend/src/connectionWorkload.test.ts | 132 ++----- packages/backend/src/connectionWorkload.ts | 152 ++++---- .../backend/src/ee/accountPermissionSyncer.ts | 2 +- packages/backend/src/jobManager.ts | 47 ++- .../backend/src/jobManagerLifecycle.test.ts | 38 +- .../src/reconciliationWorkload.test.ts | 9 +- .../backend/src/reconciliationWorkload.ts | 6 +- packages/backend/src/types.ts | 4 +- packages/backend/src/utils.ts | 2 +- .../migration.sql | 20 + packages/db/prisma/schema.prisma | 32 +- packages/shared/src/bullmqClient.test.ts | 189 ++++++++++ packages/shared/src/bullmqClient.ts | 142 ++++++++ packages/shared/src/index.server.ts | 23 +- packages/shared/src/jobLogger.test.ts | 113 ++++++ packages/shared/src/jobLogger.ts | 207 +++++++++++ packages/shared/src/jobProducer.test.ts | 90 ----- packages/shared/src/jobProducer.ts | 66 ---- packages/shared/src/queue.ts | 23 +- packages/web/src/actions.ts | 38 +- .../components/defaultSidebar/index.tsx | 5 +- .../(app)/settings/connections/[id]/page.tsx | 211 ----------- .../components/connectionJobsTable.tsx | 344 ------------------ .../components/connectionsTable.tsx | 294 --------------- .../connections/connectionSyncLogsDialog.tsx | 203 +++++++++++ .../settings/connections/connectionsList.tsx | 282 ++++++++++++++ .../app/(app)/settings/connections/page.tsx | 97 +++-- .../web/src/app/(app)/settings/layout.tsx | 7 - .../src/features/workerApi/actions.test.ts | 132 ------- .../web/src/features/workerApi/actions.ts | 45 ++- packages/web/src/lib/bullmqClient.ts | 12 + packages/web/src/lib/jobProducer.ts | 11 - 32 files changed, 1485 insertions(+), 1493 deletions(-) create mode 100644 packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql create mode 100644 packages/shared/src/bullmqClient.test.ts create mode 100644 packages/shared/src/bullmqClient.ts create mode 100644 packages/shared/src/jobLogger.test.ts create mode 100644 packages/shared/src/jobLogger.ts delete mode 100644 packages/shared/src/jobProducer.test.ts delete mode 100644 packages/shared/src/jobProducer.ts delete mode 100644 packages/web/src/app/(app)/settings/connections/[id]/page.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/connectionsList.tsx delete mode 100644 packages/web/src/features/workerApi/actions.test.ts create mode 100644 packages/web/src/lib/bullmqClient.ts delete mode 100644 packages/web/src/lib/jobProducer.ts diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index 8490118fc..d21d4a78a 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -1,8 +1,6 @@ -import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ - connectionSyncJobCreateMany: vi.fn(), - connectionSyncJobUpdate: vi.fn(), connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), transactionConnectionUpdate: vi.fn(), @@ -24,6 +22,7 @@ vi.mock('@sourcebot/shared', () => ({ attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: 500, }, }, createLogger: vi.fn(() => ({ @@ -39,10 +38,6 @@ vi.mock('@sourcebot/shared', () => ({ vi.mock('./prisma.js', () => ({ prisma: { - connectionSyncJob: { - createMany: mocks.connectionSyncJobCreateMany, - update: mocks.connectionSyncJobUpdate, - }, connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, @@ -86,62 +81,26 @@ const lifecycleContext = { maxAttempts: 2, }; -describe('connectionWorkload lifecycle', () => { +describe('connectionWorkload', () => { beforeEach(() => { vi.clearAllMocks(); - vi.useFakeTimers(); - vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); - mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); - mocks.connectionSyncJobUpdate.mockResolvedValue({}); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - test('creates a pending ConnectionSyncJob when enqueued', async () => { - await connectionWorkload.queueSpec.onEnqueued?.(lifecycleContext); - - expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ - data: [{ - id: 'job-1', - connectionId: 42, - status: 'PENDING', - warningMessages: [], - }], - skipDuplicates: true, - }); }); - test('marks the ConnectionSyncJob in progress when processing starts', async () => { - await connectionWorkload.onStarted?.(lifecycleContext); - - expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ - data: [{ - id: 'job-1', - connectionId: 42, - status: 'IN_PROGRESS', - warningMessages: [], - }], - skipDuplicates: true, - }); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - status: 'IN_PROGRESS', - }, - }); + test('does not declare database-backed lifecycle hooks', () => { + expect(connectionWorkload.queueSpec.onEnqueued).toBeUndefined(); + expect(connectionWorkload.onStarted).toBeUndefined(); + expect(connectionWorkload.onCompleted).toBeUndefined(); + expect(connectionWorkload.onTerminalFailure).toBeUndefined(); }); - test('persists compile warnings while processing', async () => { + test('discovers repositories using the connection provider', async () => { + const config = { + type: 'github' as const, + }; mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, name: 'github', - config: { - type: 'github', - }, + config, }); mocks.compileGithubConfig.mockResolvedValue({ repoData: [], @@ -150,54 +109,33 @@ describe('connectionWorkload lifecycle', () => { mocks.connectionUpdate.mockResolvedValue({}); mocks.loadConfig.mockResolvedValue({ contexts: undefined }); mocks.syncSearchContexts.mockResolvedValue(undefined); - - await connectionWorkload.process({ + const updateProgress = vi.fn(); + const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), + }; + const signal = new AbortController().signal; + + const result = await connectionWorkload.process({ ...lifecycleContext, - signal: new AbortController().signal, - log: vi.fn(), - updateProgress: vi.fn(), + signal, + logger, + updateProgress, trigger: vi.fn(), }); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - warningMessages: ['Repository was archived'], - }, - }); - }); - - test('marks the ConnectionSyncJob completed', async () => { - await connectionWorkload.onCompleted?.(lifecycleContext, undefined); - - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - status: 'COMPLETED', - completedAt: new Date('2026-07-27T12:00:00.000Z'), + expect(mocks.compileGithubConfig).toHaveBeenCalledWith(config, 42, signal); + expect(logger.info).toHaveBeenCalledWith( + 'Discovered 0 repositories', + { + connectionId: 42, + repositoryCount: 0, }, - }); - }); - - test('marks the ConnectionSyncJob failed with its error', async () => { - await connectionWorkload.onTerminalFailure?.( - { ...lifecycleContext, attemptsMade: 2 }, - new Error('Connection failed'), ); - - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: 'job-1', - }, - data: { - status: 'FAILED', - completedAt: new Date('2026-07-27T12:00:00.000Z'), - errorMessage: 'Connection failed', - }, - }); + expect(updateProgress).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); }); }); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index e81b9c954..b05b9537e 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -1,14 +1,11 @@ import { Workload } from "./types.js"; import { prisma } from "./prisma.js"; -import { ConnectionSyncJobStatus } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { CONNECTION_QUEUE, createLogger, env, loadConfig } from "@sourcebot/shared"; +import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; -const logger = createLogger('connection-workflow'); - export const connectionWorkload: Workload<'connection'> = { queueSpec: CONNECTION_QUEUE, concurrency: 2, @@ -17,9 +14,13 @@ export const connectionWorkload: Workload<'connection'> = { connectionId, orgId }, - jobId, + logger, signal, }) => { + logger.info(`Syncing connection ${connectionId}`, { + connectionId, + orgId, + }); const connection = await prisma.connection.findUniqueOrThrow({ where: { id: connectionId @@ -28,41 +29,17 @@ export const connectionWorkload: Workload<'connection'> = { const config = connection.config as unknown as ConnectionConfig; - const result = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, connectionId, signal); - } - case 'gitlab': { - return await compileGitlabConfig(config, connectionId); - } - case 'gitea': { - return await compileGiteaConfig(config, connectionId); - } - case 'gerrit': { - return await compileGerritConfig(config, connectionId); - } - case 'bitbucket': { - return await compileBitbucketConfig(config, connectionId); - } - case 'azuredevops': { - return await compileAzureDevOpsConfig(config, connectionId); - } - case 'git': { - return await compileGenericGitHostConfig(config, connectionId); - } - } - })(); + const result = await discoverConnectionRepositories({ + config, + connectionId, + signal, + }); - let { repoData, warnings } = result; + let { repoData } = result; - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, + logger.info(`Discovered ${repoData.length} repositories`, { + connectionId, + repositoryCount: repoData.length, }); // Filter out any duplicates by external_id and external_codeHostUrl. @@ -91,7 +68,11 @@ export const connectionWorkload: Workload<'connection'> = { } }); const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted all RepoToConnection records for connection ${connection.name} (id: ${connectionId}) in ${deleteDuration}ms`); + logger.debug(`Deleted existing repository associations`, { + connectionId, + connectionName: connection.name, + durationMs: deleteDuration, + }); const totalUpsertStart = performance.now(); for (const repo of repoData) { @@ -108,10 +89,19 @@ export const connectionWorkload: Workload<'connection'> = { create: repo, }) const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); + logger.debug(`Upserted repository ${repo.displayName}`, { + connectionId, + externalId: repo.external_id, + durationMs: upsertDuration, + }); } const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.debug(`Upserted ${repoData.length} repos for connection ${connection.name} (id: ${connectionId}) in ${totalUpsertDuration}ms`); + logger.info(`Stored ${repoData.length} repositories`, { + connectionId, + connectionName: connection.name, + repositoryCount: repoData.length, + durationMs: totalUpsertDuration, + }); }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); await prisma.connection.update({ @@ -133,50 +123,46 @@ export const connectionWorkload: Workload<'connection'> = { contexts: config.contexts, }); } catch (err) { - logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); + logger.error(`Failed to sync search contexts for connection ${connectionId}`, err); Sentry.captureException(err); } - }, - onStarted: async ({ data, jobId }) => { - await prisma.connectionSyncJob.createMany({ - data: [{ - id: jobId, - connectionId: data.connectionId, - status: ConnectionSyncJobStatus.IN_PROGRESS, - warningMessages: [], - }], - skipDuplicates: true, - }); - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - }, - }); - }, - onCompleted: async ({ jobId }) => { - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - }, - }); - }, - onTerminalFailure: async ({ jobId }, error) => { - await prisma.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, + + logger.info(`Connection ${connectionId} sync finished`, { + connectionId, }); } } + +const discoverConnectionRepositories = async ({ + config, + connectionId, + signal, +}: { + config: ConnectionConfig; + connectionId: number; + signal: AbortSignal; +}) => { + switch (config.type) { + case 'github': { + return compileGithubConfig(config, connectionId, signal); + } + case 'gitlab': { + return compileGitlabConfig(config, connectionId); + } + case 'gitea': { + return compileGiteaConfig(config, connectionId); + } + case 'gerrit': { + return compileGerritConfig(config, connectionId); + } + case 'bitbucket': { + return compileBitbucketConfig(config, connectionId); + } + case 'azuredevops': { + return compileAzureDevOpsConfig(config, connectionId); + } + case 'git': { + return compileGenericGitHostConfig(config, connectionId); + } + } +}; \ No newline at end of file diff --git a/packages/backend/src/ee/accountPermissionSyncer.ts b/packages/backend/src/ee/accountPermissionSyncer.ts index 88f3e57b5..14facb324 100644 --- a/packages/backend/src/ee/accountPermissionSyncer.ts +++ b/packages/backend/src/ee/accountPermissionSyncer.ts @@ -443,4 +443,4 @@ export class AccountPermissionSyncer { logger.error(errorMessage('unknown account (id not found)', 'unknown user (id not found)')); } } -} \ No newline at end of file +} diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 31e935b1f..cd49b5c76 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,9 +1,10 @@ import * as Sentry from "@sentry/node"; -import { BullMQJobProducer, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; +import { BullMQClient, createBullMQJobLogger, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; import { Job, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; +import { prisma } from "./prisma.js"; const LOG_TAG = 'job-manager'; const logger = createLogger(LOG_TAG); @@ -47,11 +48,11 @@ const scheduleToRepeat = (schedule: Schedule) => export class BullMQJobManager implements JobManager { private readonly workloads = new Map>(); private readonly workers = new Map(); - private readonly producer: BullMQJobProducer; + private readonly bullmqClient: BullMQClient; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { - this.producer = new BullMQJobProducer(connection); + this.bullmqClient = new BullMQClient(connection, prisma); } register(workload: Workload): void { @@ -85,7 +86,7 @@ export class BullMQJobManager implements JobManager { if (!workload) { throw new Error(`Cannot trigger unknown workload "${workloadName}"`); } - return this.producer.enqueue(workload.queueSpec, data); + return this.bullmqClient.enqueue(workload.queueSpec, data); } async stop(): Promise { @@ -98,7 +99,7 @@ export class BullMQJobManager implements JobManager { ]), )); - await this.producer.close(); + await this.bullmqClient.close(); logger.info('Job manager stopped'); } @@ -106,21 +107,35 @@ export class BullMQJobManager implements JobManager { private async startWorkload(workload: Workload): Promise { const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; - const queue = this.producer.queue(spec.name); + const queue = this.bullmqClient.getQueue(spec); const worker = new Worker( spec.name, async (job) => { + const jobLogger = createBullMQJobLogger( + job, + `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, + ); const lifecycleContext = this.jobLifecycleContext(job); - await workload.onStarted?.(lifecycleContext); - - return workload.process({ - ...lifecycleContext, - signal: this.abortController.signal, - log: async (message) => { await job.log(message); }, - updateProgress: (progress) => job.updateProgress(progress), - trigger: (target, data) => this.trigger(target, data), - }); + jobLogger.info(`Started workload "${spec.name}"`); + + try { + await workload.onStarted?.(lifecycleContext); + const result = await workload.process({ + ...lifecycleContext, + signal: this.abortController.signal, + logger: jobLogger, + updateProgress: (progress) => job.updateProgress(progress), + trigger: (target, data) => this.trigger(target, data), + }); + jobLogger.info(`Completed workload "${spec.name}"`); + return result; + } catch (error) { + jobLogger.error(`Workload "${spec.name}" attempt failed`, error); + throw error; + } finally { + await jobLogger.flush(); + } }, { connection: this.connection, @@ -159,6 +174,7 @@ export class BullMQJobManager implements JobManager { attempts: spec.jobOptions.attempts, removeOnComplete: { count: spec.jobOptions.keep.completed }, removeOnFail: { count: spec.jobOptions.keep.failed }, + keepLogs: spec.jobOptions.keepLogs, }, }, ); @@ -208,6 +224,7 @@ export class BullMQJobManager implements JobManager { jobId: job.id ?? '', attemptsMade: job.attemptsMade, maxAttempts: job.opts.attempts ?? 1, + prisma, }; } } diff --git a/packages/backend/src/jobManagerLifecycle.test.ts b/packages/backend/src/jobManagerLifecycle.test.ts index 5117233b3..86a1120f4 100644 --- a/packages/backend/src/jobManagerLifecycle.test.ts +++ b/packages/backend/src/jobManagerLifecycle.test.ts @@ -1,11 +1,18 @@ import { Redis } from 'ioredis'; import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { Workload } from './types.js'; +import { ProcessContext, Workload } from './types.js'; const mocks = vi.hoisted(() => ({ enqueue: vi.fn(), producerClose: vi.fn(), workerClose: vi.fn(), + jobLogger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + flush: vi.fn(), + }, workers: [] as Array<{ processor: (job: unknown) => Promise; handlers: Map void>; @@ -23,7 +30,8 @@ vi.mock('@sourcebot/shared', () => ({ error: vi.fn(), debug: vi.fn(), })), - BullMQJobProducer: class { + createBullMQJobLogger: vi.fn(() => mocks.jobLogger), + BullMQClient: class { enqueue = mocks.enqueue; close = mocks.producerClose; queue = vi.fn(() => ({ @@ -66,6 +74,7 @@ const createWorkload = ( attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: 500, }, }, concurrency: 2, @@ -90,7 +99,7 @@ describe('BullMQJobManager lifecycle', () => { mocks.enqueue.mockResolvedValue('job-1'); }); - test('delegates enqueueing to BullMQJobProducer and returns its job id', async () => { + test('delegates enqueueing to BullMQClient and returns its job id', async () => { const manager = new BullMQJobManager({} as Redis); const workload = createWorkload(); manager.register(workload); @@ -124,6 +133,25 @@ describe('BullMQJobManager lifecycle', () => { expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), { repoCount: 3 }, ); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test('provides the structured job logger to the workload processor', async () => { + const process = vi.fn(async (context: ProcessContext<'connection'>) => { + context.logger.info('Processing connection'); + return { repoCount: 3 }; + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload({ process })); + await manager.start(); + + await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + + expect(process).toHaveBeenCalledWith(expect.objectContaining({ + logger: mocks.jobLogger, + })); + expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); }); test('reports lifecycle metadata after terminal failure', async () => { @@ -137,12 +165,12 @@ describe('BullMQJobManager lifecycle', () => { mocks.workers[0].handlers.get('failed')?.(job, error); await vi.waitFor(() => { - expect(onTerminalFailure).toHaveBeenCalledWith({ + expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ data, jobId: 'job-1', attemptsMade: 2, maxAttempts: 2, - }, error); + }), error); }); }); }); diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index 61db94afb..3aea21082 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -8,6 +8,7 @@ vi.mock('@sourcebot/shared', () => ({ attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: 500, }, }, createLogger: vi.fn(() => ({ @@ -67,7 +68,13 @@ describe('reconciliationWorkload', () => { attemptsMade: 0, maxAttempts: 2, signal: new AbortController().signal, - log: vi.fn(), + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), + }, updateProgress: vi.fn(), trigger, }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index 500389b2e..73209b26c 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -1,9 +1,7 @@ import { PrismaClient } from "@sourcebot/db"; -import { createLogger, RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { RECONCILIATION_QUEUE } from "@sourcebot/shared"; import { Settings, Workload } from "./types.js"; -const logger = createLogger('reconciliation-workload'); - interface ReconciliationWorkloadDependencies { db: PrismaClient; settings: Pick; @@ -16,7 +14,7 @@ export const createReconciliationWorkload = ({ concurrency: 1, schedule: { every: '15m' }, queueSpec: RECONCILIATION_QUEUE, - process: async ({ trigger }) => { + process: async ({ logger, trigger }) => { const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); const connections = await db.connection.findMany({ where: { diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index ae6b7fc7e..848d9f48f 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,7 +1,7 @@ import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; -import { DataOf, JobLifecycleContext, QueueName, QueueSpec } from "@sourcebot/shared"; +import { DataOf, JobLifecycleContext, JobLogger, QueueName, QueueSpec } from "@sourcebot/shared"; export type Settings = Required; @@ -26,7 +26,7 @@ export type RepoAuthCredentials = { export interface ProcessContext extends JobLifecycleContext { signal: AbortSignal; - log(message: string): Promise; + logger: JobLogger; updateProgress(progress: number | object): Promise; trigger(workload: T, data: DataOf): Promise; } diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index d99727ea3..e50ee0ed5 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -284,4 +284,4 @@ export const setIntervalAsync = (target: () => Promise, pollingIntervalMs: setIntervalWithPromise(target), pollingIntervalMs ); -} \ No newline at end of file +} diff --git a/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql b/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql new file mode 100644 index 000000000..c4315cee2 --- /dev/null +++ b/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql @@ -0,0 +1,20 @@ +/* + Warnings: + + - You are about to drop the `ConnectionSyncJob` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropForeignKey +ALTER TABLE "ConnectionSyncJob" DROP CONSTRAINT "ConnectionSyncJob_connectionId_fkey"; + +-- AlterTable +ALTER TABLE "Connection" ADD COLUMN "latestSyncJobId" TEXT; + +-- DropTable +DROP TABLE "ConnectionSyncJob"; + +-- DropEnum +DROP TYPE "ConnectionSyncJobStatus"; + +-- DropEnum +DROP TYPE "ConnectionSyncStatus"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 0f72c97d0..336a306e9 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -10,15 +10,6 @@ datasource db { url = env("DATABASE_URL") } -enum ConnectionSyncStatus { - SYNC_NEEDED - IN_SYNC_QUEUE - SYNCING - SYNCED - SYNCED_WITH_WARNINGS - FAILED -} - enum ChatVisibility { PRIVATE PUBLIC @@ -181,9 +172,9 @@ model Connection { // The type of connection (e.g., github, gitlab, etc.) connectionType ConnectionType - syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? + latestSyncJobId String? /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -208,27 +199,6 @@ model Connection { @@unique([name, orgId]) } -enum ConnectionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model ConnectionSyncJob { - id String @id @default(cuid()) - status ConnectionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - warningMessages String[] - errorMessage String? - - connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) - connectionId Int -} - model RepoToConnection { addedAt DateTime @default(now()) diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts new file mode 100644 index 000000000..ee165d5b8 --- /dev/null +++ b/packages/shared/src/bullmqClient.test.ts @@ -0,0 +1,189 @@ +import type { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import type { QueueSpec } from './queue.js'; +import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; + +const queueMocks = vi.hoisted(() => ({ + add: vi.fn(), + close: vi.fn(), + getJob: vi.fn(), + getJobLogs: vi.fn(), +})); + +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('./logger.js', () => ({ + createLogger: vi.fn(() => ({ + error: vi.fn(), + })), +})); + +vi.mock('bullmq', () => ({ + Queue: class { + constructor() { + return queueMocks; + } + }, +})); + +import { BullMQClient } from './bullmqClient.js'; +import { PrismaClient } from '@sourcebot/db'; + +const connectionSpec: QueueSpec<'connection'> = { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, +}; + +const data = { connectionId: 42, orgId: 1 }; + +describe('BullMQClient', () => { + const redis = {} as Redis; + const prisma = {} as PrismaClient; + + beforeEach(() => { + vi.clearAllMocks(); + queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); + }); + + test('returns the job id when BullMQ accepts the proposed id', async () => { + const client = new BullMQClient(redis, prisma); + + const result = await client.enqueue(connectionSpec, data); + + expect(result).toEqual(expect.any(String)); + expect(queueMocks.add).toHaveBeenCalledWith( + 'connection', + data, + expect.objectContaining({ + jobId: result, + deduplication: { id: 'connection:42' }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }), + ); + }); + + test('calls onEnqueued when a new job is created', async () => { + const onEnqueued = vi.fn(); + const client = new BullMQClient(redis, prisma); + + const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(onEnqueued).toHaveBeenCalledWith({ + data, + jobId: result, + attemptsMade: 0, + maxAttempts: 2, + prisma, + }); + }); + + test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { + const onEnqueued = vi.fn(); + queueMocks.add.mockResolvedValue({ id: 'existing-job' }); + const client = new BullMQClient(redis, prisma); + + const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); + + expect(result).toBe('existing-job'); + expect(onEnqueued).not.toHaveBeenCalled(); + }); + + test.each([ + ['waiting', 'PENDING'], + ['waiting-children', 'PENDING'], + ['delayed', 'PENDING'], + ['prioritized', 'PENDING'], + ['paused', 'PENDING'], + ['active', 'IN_PROGRESS'], + ['completed', 'COMPLETED'], + ])('maps BullMQ state %s to %s', async (state, expectedStatus) => { + queueMocks.getJob.mockResolvedValue({ + id: 'job-1', + data, + getState: vi.fn().mockResolvedValue(state), + }); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ + id: 'job-1', + data, + status: expectedStatus, + errorMessage: null, + }); + }); + + test('returns the failure reason for a failed job', async () => { + queueMocks.getJob.mockResolvedValue({ + id: 'job-1', + data, + failedReason: 'Connection credentials expired', + getState: vi.fn().mockResolvedValue('failed'), + }); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ + id: 'job-1', + data, + status: 'FAILED', + errorMessage: 'Connection credentials expired', + }); + }); + + test.each([ + ['missing job', undefined], + ['unknown state', { + id: 'job-1', + data, + getState: vi.fn().mockResolvedValue('unknown'), + }], + ])('returns null for a %s', async (_label, job) => { + queueMocks.getJob.mockResolvedValue(job); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJob(connectionSpec, 'job-1')).resolves.toBeNull(); + }); + + test('reads and parses incremental job logs', async () => { + queueMocks.getJobLogs.mockResolvedValue({ + logs: [ + JSON.stringify({ + version: 1, + timestamp: '2026-07-28T12:00:00.000Z', + level: 'warn', + message: 'Repository skipped', + attempt: 1, + }), + ], + count: 4, + }); + const client = new BullMQClient(redis, prisma); + + await expect(client.getJobLogs(connectionSpec, 'job-1', { + start: 3, + ascending: true, + })).resolves.toEqual({ + logs: [{ + version: 1, + timestamp: '2026-07-28T12:00:00.000Z', + level: 'warn', + message: 'Repository skipped', + attempt: 1, + }], + count: 4, + }); + expect(queueMocks.getJobLogs).toHaveBeenCalledWith( + 'job-1', + 3, + undefined, + true, + ); + }); +}); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts new file mode 100644 index 000000000..52a6bff54 --- /dev/null +++ b/packages/shared/src/bullmqClient.ts @@ -0,0 +1,142 @@ +import * as Sentry from "@sentry/node"; +import { Queue } from "bullmq"; +import { randomUUID } from "crypto"; +import { Redis } from "ioredis"; +import { createLogger } from "./logger.js"; +import { DataOf, QueueName, QueueSpec } from "./queue.js"; +import { PrismaClient } from "@sourcebot/db"; +import { readBullMQJobLogs } from "./jobLogger.js"; +import type { GetJobLogsOptions, JobLogs } from "./jobLogger.js"; + +const logger = createLogger('job-producer'); + +export type WorkloadJobStatus = "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED"; + +export interface WorkloadJob { + id: string; + data: DataOf; + status: WorkloadJobStatus; + errorMessage: string | null; +} + +type WorkloadQueue = Queue< + DataOf, + unknown, + string, + DataOf, + unknown, + string +>; + +const normalizeJobState = (state: string): WorkloadJobStatus | null => { + switch (state) { + case "waiting": + case "waiting-children": + case "delayed": + case "prioritized": + case "paused": + return "PENDING"; + case "active": + return "IN_PROGRESS"; + case "completed": + return "COMPLETED"; + case "failed": + return "FAILED"; + default: + return null; + } +}; + +export class BullMQClient { + private readonly queues = new Map(); + + constructor( + private readonly connection: Redis, + private readonly prisma: PrismaClient, + ) {} + + getQueue(spec: QueueSpec): WorkloadQueue { + const queueName = spec.name; + let queue = this.queues.get(queueName); + if (!queue) { + queue = new Queue(queueName, { connection: this.connection }); + this.queues.set(queueName, queue); + } + return queue as WorkloadQueue; + } + + async getJob( + spec: QueueSpec, + jobId: string, + ): Promise | null> { + const job = await this.getQueue(spec).getJob(jobId); + if (!job) { + return null; + } + + const status = normalizeJobState(await job.getState()); + if (!status) { + return null; + } + + return { + id: job.id ?? jobId, + data: job.data as DataOf, + status, + errorMessage: status === "FAILED" ? job.failedReason || null : null, + }; + } + + async getJobLogs( + spec: QueueSpec, + jobId: string, + options: GetJobLogsOptions = {}, + ): Promise { + return readBullMQJobLogs(this.getQueue(spec), jobId, options); + } + + async enqueue( + spec: QueueSpec, + data: DataOf + ): Promise { + const dedupKey = spec.dedupKey?.(data); + const queue = this.getQueue(spec); + + const requestedJobId = randomUUID(); + const job = await queue.add(spec.name, data, { + jobId: requestedJobId, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + attempts: spec.jobOptions.attempts, + backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + keepLogs: spec.jobOptions.keepLogs, + }); + + if (!job.id) { + throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); + } + + const isEnqueued = job.id === requestedJobId; + if (isEnqueued && spec.onEnqueued) { + try { + await spec.onEnqueued({ + data, + jobId: job.id, + attemptsMade: 0, + maxAttempts: spec.jobOptions.attempts, + prisma: this.prisma, + }); + } catch (error) { + Sentry.captureException(error); + logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); + } + } + + return job.id; + } + + async close(): Promise { + await Promise.all([...this.queues.values()].map((queue) => queue.close())); + } +} diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 753240ab5..51d4c9077 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -103,5 +103,24 @@ export { RECONCILIATION_QUEUE, } from "./queue.js"; export { - BullMQJobProducer, -} from "./jobProducer.js"; + BullMQClient, +} from "./bullmqClient.js"; +export type { + WorkloadJob, + WorkloadJobStatus, +} from "./bullmqClient.js"; +export { + createBullMQJobLogger, + DEFAULT_JOB_LOGS_MAX_ENTRIES, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; +export type { + GetJobLogsOptions, + JobLogEntry, + JobLogFields, + JobLogLevel, + JobLogger, + JobLogs, + JobLogSink, +} from "./jobLogger.js"; diff --git a/packages/shared/src/jobLogger.test.ts b/packages/shared/src/jobLogger.test.ts new file mode 100644 index 000000000..c6435ea00 --- /dev/null +++ b/packages/shared/src/jobLogger.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + applicationLog: vi.fn(), + applicationError: vi.fn(), +})); + +vi.mock("./logger.js", () => ({ + createLogger: vi.fn(() => ({ + log: mocks.applicationLog, + error: mocks.applicationError, + })), +})); + +import { + createBullMQJobLogger, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; + +describe("createBullMQJobLogger", () => { + test("writes structured, redacted entries to BullMQ and the application logger", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 1, + log, + }); + + logger.warn("Some repositories were skipped", { + skipped: 2, + accessToken: "do-not-store", + }); + await logger.flush(); + + expect(mocks.applicationLog).toHaveBeenCalledWith( + "warn", + "Some repositories were skipped", + { + skipped: 2, + accessToken: "[REDACTED]", + }, + ); + + const storedEntry = JSON.parse(log.mock.calls[0][0]); + expect(storedEntry).toMatchObject({ + version: 1, + level: "warn", + message: "Some repositories were skipped", + attempt: 2, + fields: { + skipped: 2, + accessToken: "[REDACTED]", + }, + }); + expect(storedEntry.timestamp).toEqual(expect.any(String)); + }); + + test("does not fail the workload when persisting a log entry fails", async () => { + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 0, + log: vi.fn().mockRejectedValue(new Error("Redis unavailable")), + }); + + logger.info("Starting"); + await expect(logger.flush()).resolves.toBeUndefined(); + expect(mocks.applicationError).toHaveBeenCalled(); + }); +}); + +describe("readBullMQJobLogs", () => { + test("parses structured entries and preserves legacy string logs", async () => { + const structuredEntry = JSON.stringify({ + version: 1, + timestamp: "2026-07-28T03:00:00.000Z", + level: "info", + message: "Started", + attempt: 1, + }); + const queue = { + getJobLogs: vi.fn().mockResolvedValue({ + logs: [structuredEntry, "legacy log"], + count: 2, + }), + }; + + const result = await readBullMQJobLogs(queue, "job-1", { + start: 10, + end: 20, + ascending: true, + }); + + expect(queue.getJobLogs).toHaveBeenCalledWith("job-1", 10, 20, true); + expect(result).toEqual({ + logs: [ + parseJobLogEntry(structuredEntry), + { + version: 0, + timestamp: null, + level: "info", + message: "legacy log", + attempt: null, + }, + ], + count: 2, + }); + }); +}); diff --git a/packages/shared/src/jobLogger.ts b/packages/shared/src/jobLogger.ts new file mode 100644 index 000000000..57df9311b --- /dev/null +++ b/packages/shared/src/jobLogger.ts @@ -0,0 +1,207 @@ +import type { Job, Queue } from "bullmq"; +import { createLogger } from "./logger.js"; + +export const DEFAULT_JOB_LOGS_MAX_ENTRIES = 500; + +export type JobLogLevel = "debug" | "info" | "warn" | "error"; +export type JobLogFields = Record; + +export interface JobLogEntry { + version: 1 | 0; + timestamp: string | null; + level: JobLogLevel; + message: string; + attempt: number | null; + fields?: JobLogFields; +} + +export interface JobLogSink { + debug(message: string, fields?: unknown): void; + info(message: string, fields?: unknown): void; + warn(message: string, fields?: unknown): void; + error(message: string, fields?: unknown): void; +} + +export interface JobLogger extends JobLogSink { + flush(): Promise; +} + +export interface GetJobLogsOptions { + start?: number; + end?: number; + ascending?: boolean; +} + +export interface JobLogs { + logs: JobLogEntry[]; + count: number; +} + +type BullMQLogJob = Pick; +type JobLogQueue = Pick; + +const JOB_LOG_LEVELS = new Set(["debug", "info", "warn", "error"]); +const SENSITIVE_FIELD_NAME = /authorization|cookie|credential|password|private.?key|secret|token/i; +const MAX_FIELD_DEPTH = 6; + +const sanitizeValue = ( + value: unknown, + seen: WeakSet, + depth: number, +): unknown => { + if (depth > MAX_FIELD_DEPTH) { + return "[Max depth reached]"; + } + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "bigint") { + return value.toString(); + } + if (typeof value === "undefined") { + return "[undefined]"; + } + if (typeof value === "symbol" || typeof value === "function") { + return String(value); + } + if (value instanceof Date) { + return value.toISOString(); + } + if (value instanceof Error) { + return { + name: value.name, + message: value.message, + stack: value.stack, + }; + } + if (seen.has(value)) { + return "[Circular]"; + } + + seen.add(value); + if (Array.isArray(value)) { + return value.map((item) => sanitizeValue(item, seen, depth + 1)); + } + + return Object.fromEntries( + Object.entries(value).map(([key, nestedValue]) => [ + key, + SENSITIVE_FIELD_NAME.test(key) + ? "[REDACTED]" + : sanitizeValue(nestedValue, seen, depth + 1), + ]), + ); +}; + +const sanitizeFields = (fields: unknown): JobLogFields | undefined => { + if (fields === undefined) { + return undefined; + } + + const sanitized = sanitizeValue(fields, new WeakSet(), 0); + if (sanitized !== null && typeof sanitized === "object" && !Array.isArray(sanitized)) { + return sanitized as JobLogFields; + } + return { value: sanitized }; +}; + +export const parseJobLogEntry = (rawLog: string): JobLogEntry => { + try { + const parsed = JSON.parse(rawLog) as Partial; + if ( + parsed.version === 1 && + typeof parsed.timestamp === "string" && + typeof parsed.level === "string" && + JOB_LOG_LEVELS.has(parsed.level as JobLogLevel) && + typeof parsed.message === "string" && + typeof parsed.attempt === "number" + ) { + return { + version: 1, + timestamp: parsed.timestamp, + level: parsed.level as JobLogLevel, + message: parsed.message, + attempt: parsed.attempt, + ...(parsed.fields ? { fields: parsed.fields } : {}), + }; + } + } catch { + // Older BullMQ logs were stored as plain strings. + } + + return { + version: 0, + timestamp: null, + level: "info", + message: rawLog, + attempt: null, + }; +}; + +export const readBullMQJobLogs = async ( + queue: JobLogQueue, + jobId: string, + options: GetJobLogsOptions = {}, +): Promise => { + const result = await queue.getJobLogs( + jobId, + options.start, + options.end, + options.ascending, + ); + + return { + logs: result.logs.map(parseJobLogEntry), + count: result.count, + }; +}; + +export const createBullMQJobLogger = ( + job: BullMQLogJob, + label = `${job.queueName}:job:${job.id ?? "unknown"}`, +): JobLogger => { + const applicationLogger = createLogger(label); + const pendingWrites = new Set>(); + + const write = (level: JobLogLevel, message: string, rawFields?: unknown): void => { + const fields = sanitizeFields(rawFields); + applicationLogger.log(level, message, fields); + + const entry: JobLogEntry = { + version: 1, + timestamp: new Date().toISOString(), + level, + message, + attempt: job.attemptsMade + 1, + ...(fields ? { fields } : {}), + }; + const pendingWrite = job.log(JSON.stringify(entry)) + .then(() => undefined) + .catch((error: unknown) => { + applicationLogger.error( + `Failed to persist a BullMQ log entry for job ${job.id ?? "unknown"}`, + error, + ); + }); + + pendingWrites.add(pendingWrite); + void pendingWrite.finally(() => { + pendingWrites.delete(pendingWrite); + }); + }; + + return { + debug: (message, fields) => write("debug", message, fields), + info: (message, fields) => write("info", message, fields), + warn: (message, fields) => write("warn", message, fields), + error: (message, fields) => write("error", message, fields), + flush: async () => { + await Promise.all([...pendingWrites]); + }, + }; +}; diff --git a/packages/shared/src/jobProducer.test.ts b/packages/shared/src/jobProducer.test.ts deleted file mode 100644 index ffdad845b..000000000 --- a/packages/shared/src/jobProducer.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import type { QueueSpec } from './queue.js'; - -const queueMocks = vi.hoisted(() => ({ - add: vi.fn(), - close: vi.fn(), -})); - -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('./logger.js', () => ({ - createLogger: vi.fn(() => ({ - error: vi.fn(), - })), -})); - -vi.mock('bullmq', () => ({ - Queue: class { - constructor() { - return queueMocks; - } - }, -})); - -import { BullMQJobProducer } from './jobProducer.js'; - -const connectionSpec: QueueSpec<'connection'> = { - name: 'connection', - dedupKey: ({ connectionId }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - }, -}; - -const data = { connectionId: 42, orgId: 1 }; - -describe('BullMQJobProducer', () => { - const redis = {} as Redis; - - beforeEach(() => { - vi.clearAllMocks(); - queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); - }); - - test('returns the job id when BullMQ accepts the proposed id', async () => { - const producer = new BullMQJobProducer(redis); - - const result = await producer.enqueue(connectionSpec, data); - - expect(result).toEqual(expect.any(String)); - expect(queueMocks.add).toHaveBeenCalledWith( - 'connection', - data, - expect.objectContaining({ - jobId: result, - deduplication: { id: 'connection:42' }, - }), - ); - }); - - test('calls onEnqueued when a new job is created', async () => { - const onEnqueued = vi.fn(); - const producer = new BullMQJobProducer(redis); - - const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(onEnqueued).toHaveBeenCalledWith({ - data, - jobId: result, - attemptsMade: 0, - maxAttempts: 2, - }); - }); - - test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { - const onEnqueued = vi.fn(); - queueMocks.add.mockResolvedValue({ id: 'existing-job' }); - const producer = new BullMQJobProducer(redis); - - const result = await producer.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(result).toBe('existing-job'); - expect(onEnqueued).not.toHaveBeenCalled(); - }); -}); diff --git a/packages/shared/src/jobProducer.ts b/packages/shared/src/jobProducer.ts deleted file mode 100644 index 595a46787..000000000 --- a/packages/shared/src/jobProducer.ts +++ /dev/null @@ -1,66 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { Queue } from "bullmq"; -import { randomUUID } from "crypto"; -import { Redis } from "ioredis"; -import { createLogger } from "./logger.js"; -import { DataOf, QueueName, QueueSpec } from "./queue.js"; - -const logger = createLogger('job-producer'); - -export class BullMQJobProducer { - private readonly queues = new Map(); - - constructor(private readonly connection: Redis) {} - - queue(name: string): Queue { - let queue = this.queues.get(name); - if (!queue) { - queue = new Queue(name, { connection: this.connection }); - this.queues.set(name, queue); - } - return queue; - } - - async enqueue( - spec: QueueSpec, - data: DataOf - ): Promise { - const dedupKey = spec.dedupKey?.(data); - const queue = this.queue(spec.name); - - const requestedJobId = randomUUID(); - const job = await queue.add(spec.name, data, { - jobId: requestedJobId, - ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), - attempts: spec.jobOptions.attempts, - backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, - removeOnComplete: { count: spec.jobOptions.keep.completed }, - removeOnFail: { count: spec.jobOptions.keep.failed }, - }); - - if (!job.id) { - throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); - } - - const isEnqueued = job.id === requestedJobId; - if (isEnqueued && spec.onEnqueued) { - try { - await spec.onEnqueued({ - data, - jobId: job.id, - attemptsMade: 0, - maxAttempts: spec.jobOptions.attempts, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); - } - } - - return job.id; - } - - async close(): Promise { - await Promise.all([...this.queues.values()].map((queue) => queue.close())); - } -} diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index ef9dd67d5..fb375d711 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,4 +1,7 @@ +import { PrismaClient } from "@sourcebot/db"; +import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; + export type QueueName = keyof QueueRegistry; export type DataOf = QueueRegistry[TName]; type EmptyJobData = Record; @@ -16,7 +19,22 @@ export const CONNECTION_QUEUE: QueueSpec<'connection'> = { jobOptions: { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 } + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + onEnqueued: async ({ + prisma, + data: { connectionId }, + jobId + }) => { + await prisma.connection.update({ + where: { + id: connectionId + }, + data: { + latestSyncJobId: jobId + } + }); }, dedupKey: (data) => `connection:${data.connectionId}`, } @@ -28,6 +46,7 @@ export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, }; @@ -38,6 +57,7 @@ export interface QueueSpec { attempts: number; backoff: { type: 'fixed' | 'exponential'; delayMs: number }; keep: { completed: number; failed: number }; + keepLogs: number; }; onEnqueued?(ctx: JobLifecycleContext): Promise; } @@ -47,4 +67,5 @@ export interface JobLifecycleContext { jobId: string; attemptsMade: number; maxAttempts: number; + prisma: PrismaClient; } diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index c64979173..5db1dbdfc 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, ServiceError } from "@/lib/serviceError"; import { sew } from "@/middleware/sew"; -import { ConnectionSyncJobStatus, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; @@ -274,42 +274,6 @@ export const getReposStats = async () => sew(() => }) ) -export const getConnectionStats = async () => sew(() => - withAuth(async ({ org, prisma }) => { - const [ - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - ] = await Promise.all([ - prisma.connection.count({ - where: { - orgId: org.id, - } - }), - prisma.connection.count({ - where: { - orgId: org.id, - syncedAt: null, - syncJobs: { - some: { - status: { - in: [ - ConnectionSyncJobStatus.PENDING, - ConnectionSyncJobStatus.IN_PROGRESS, - ] - } - } - } - } - }) - ]); - - return { - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - }; - }) -); - export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuth(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx index 04ff3caf6..62a6c0256 100644 --- a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx +++ b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx @@ -2,7 +2,6 @@ import { cookies } from "next/headers"; import { auth } from "@/auth"; import { HOME_VIEW_COOKIE_NAME } from "@/lib/constants"; import { HomeView } from "@/hooks/useHomeView"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; @@ -45,11 +44,9 @@ export async function DefaultSidebar() { if (!isOwner) { return false; } - const connectionStats = await getConnectionStats(); const joinRequests = await getOrgAccountRequests(); - const hasConnectionNotification = !isServiceError(connectionStats) && connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0; const hasJoinRequestNotification = !isServiceError(joinRequests) && joinRequests.length > 0; - return hasConnectionNotification || hasJoinRequestNotification; + return hasJoinRequestNotification; })(); return ( diff --git a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx deleted file mode 100644 index edcc61069..000000000 --- a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx +++ /dev/null @@ -1,211 +0,0 @@ -import { sew } from "@/middleware/sew"; -import { BackButton } from "@/app/(app)/components/backButton"; -import { DisplayDate } from "@/app/(app)/components/DisplayDate"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Skeleton } from "@/components/ui/skeleton"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { notFound as notFoundServiceError, ServiceErrorException } from "@/lib/serviceError"; -import { notFound } from "next/navigation"; -import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; -import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; -import { env, getConfigSettings } from "@sourcebot/shared"; -import { Info } from "lucide-react"; -import Link from "next/link"; -import { Suspense } from "react"; -import { ConnectionJobsTable } from "../components/connectionJobsTable"; - -interface ConnectionDetailPageProps { - params: Promise<{ - id: string - }> -} - -export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { - const params = await props.params; - const { id } = params; - - const connectionId = Number.parseInt(id); - if (isNaN(connectionId)) { - return notFound(); - } - - const connection = await getConnectionWithJobs(connectionId); - if (isServiceError(connection)) { - throw new ServiceErrorException(connection); - } - - const configSettings = await getConfigSettings(env.CONFIG_PATH); - - const nextSyncAttempt = (() => { - const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null; - if (!latestJob) { - return undefined; - } - - if (latestJob.completedAt) { - return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs); - } - - return undefined; - })(); - - // Extracts the code host URL from the connection config. - const codeHostUrl: string = (() => { - const connectionType = connection.connectionType; - switch (connectionType) { - case 'github': { - const config = connection.config as unknown as GithubConnectionConfig; - return config.url ?? 'https://github.com'; - } - case 'gitlab': { - const config = connection.config as unknown as GitlabConnectionConfig; - return config.url ?? 'https://gitlab.com'; - } - case 'gitea': { - const config = connection.config as unknown as GiteaConnectionConfig; - return config.url ?? 'https://gitea.com'; - } - case 'gerrit': { - const config = connection.config as unknown as GerritConnectionConfig; - return config.url; - } - case 'bitbucket': { - const config = connection.config as unknown as BitbucketConnectionConfig; - if (config.deploymentType === 'cloud') { - return config.url ?? 'https://bitbucket.org'; - } else { - return config.url!; - } - } - case 'azuredevops': { - const config = connection.config as unknown as AzureDevOpsConnectionConfig; - return config.url ?? 'https://dev.azure.com'; - } - case 'git': { - const config = connection.config as unknown as GenericGitHostConnectionConfig; - return config.url; - } - } - })(); - - return ( -
- -
-

{connection.name}

- - - {codeHostUrl} - -
- -
- - - - Created - - - - - -

When this connection was first added to Sourcebot

-
-
-
-
- - - -
- - - - - Last synced - - - - - -

The last time this connection was successfully synced

-
-
-
-
- - {connection.syncedAt ? : "Never"} - -
- - - - - Scheduled - - - - - -

When the connection will be resynced next. Modifying the config will also trigger a resync.

-
-
-
-
- - {nextSyncAttempt ? : "-"} - -
-
- - - - Sync History - History of all sync jobs for this connection. - - - }> - - - - -
- ) -} - -const getConnectionWithJobs = async (id: number) => sew(() => - withAuth(async ({ prisma, org }) => { - const connection = await prisma.connection.findUnique({ - where: { - id, - orgId: org.id, - }, - include: { - syncJobs: { - orderBy: { - createdAt: 'desc', - }, - }, - }, - }); - - if (!connection) { - return notFoundServiceError(); - } - - return connection; - }) -) \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx deleted file mode 100644 index 9277d81b7..000000000 --- a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx +++ /dev/null @@ -1,344 +0,0 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { AlertCircle, AlertTriangle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" -import * as React from "react" -import { CopyIconButton } from "@/app/(app)/components/copyIconButton" -import { useMemo } from "react" -import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { LoadingButton } from "@/components/ui/loading-button" -import { syncConnection } from "@/features/workerApi/actions" -import { isServiceError } from "@/lib/utils" - - -export type ConnectionSyncJob = { - id: string - status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" - createdAt: Date - updatedAt: Date - completedAt: Date | null - errorMessage: string | null - warningMessages: string[] -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: ConnectionSyncJob["status"]) => { - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -const getDuration = (start: Date, end: Date | null) => { - if (!end) return "-" - const diff = end.getTime() - start.getTime() - const minutes = Math.floor(diff / 60000) - const seconds = Math.floor((diff % 60000) / 1000) - return `${minutes}m ${seconds}s` -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "status", - header: "Status", - cell: ({ row }) => { - const job = row.original - return ( -
- {getStatusBadge(row.getValue("status"))} - {job.errorMessage ? ( - - - - - - - - {job.errorMessage} - - - - - ) : job.warningMessages.length > 0 ? ( - - - - - - -

{job.warningMessages.length} warning(s) while syncing:

-
- {job.warningMessages.map((warning, index) => ( -
- {index + 1}. - {warning} -
- ))} -
-
-
-
- ) : null} -
- ) - }, - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "createdAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => , - }, - { - accessorKey: "completedAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const completedAt = row.getValue("completedAt") as Date | null; - if (!completedAt) { - return "-"; - } - - return - }, - }, - { - id: "duration", - header: "Duration", - cell: ({ row }) => { - const job = row.original - return getDuration(job.createdAt, job.completedAt) - }, - }, - { - accessorKey: "id", - header: "Job ID", - cell: ({ row }) => { - const id = row.getValue("id") as string - return ( -
- {id} - { - navigator.clipboard.writeText(id); - return true; - }} /> -
- ) - }, - }, -] - -export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSyncJob[], connectionId: number }) => { - const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) - const [columnFilters, setColumnFilters] = React.useState([]) - const [columnVisibility, setColumnVisibility] = React.useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const [isSyncSubmitting, setIsSyncSubmitting] = React.useState(false); - const onSyncButtonClick = React.useCallback(async () => { - setIsSyncSubmitting(true); - const response = await syncConnection(connectionId); - - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Connection synced successfully. Job ID: ${jobId}`, - }) - router.refresh(); - } else { - toast({ - description: `❌ Failed to sync connection. ${response.message}`, - }); - } - - setIsSyncSubmitting(false); - }, [connectionId, router, toast]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - state: { - sorting, - columnFilters, - columnVisibility, - }, - }) - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - } = useMemo(() => { - return { - numCompleted: data.filter((job) => job.status === "COMPLETED").length, - numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, - numPending: data.filter((job) => job.status === "PENDING").length, - numFailed: data.filter((job) => job.status === "FAILED").length, - }; - }, [data]); - - return ( -
-
- - -
- - - - - Trigger sync - -
-
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No sync jobs found. - - - )} - -
-
- -
-
- {table.getFilteredRowModel().rows.length} job(s) total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx deleted file mode 100644 index e52e7ef1e..000000000 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx +++ /dev/null @@ -1,294 +0,0 @@ -"use client" - -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { NotificationDot } from "@/app/(app)/components/notificationDot" -import { useToast } from "@/components/hooks/use-toast" -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { getCodeHostIcon } from "@/lib/utils" -import { ConnectionType } from "@sourcebot/db" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { ArrowUpDown, RefreshCwIcon } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { useRouter } from "next/navigation" -import { useMemo, useState } from "react" - - -export type Connection = { - id: number - name: string - syncedAt: Date | null - connectionType: ConnectionType - latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null - isFirstTimeSync: boolean -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: Connection["latestJobStatus"]) => { - if (!status) { - return "-"; - } - - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "name", - size: 400, - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const connection = row.original; - const codeHostIcon = getCodeHostIcon(connection.connectionType); - - return ( -
- {`${connection.connectionType} - - {connection.name} - - {connection.isFirstTimeSync && ( - - - - - - - - This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete. - - - )} -
- ) - }, - }, - { - accessorKey: "latestJobStatus", - size: 150, - header: "Lastest status", - cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), - }, - { - accessorKey: "syncedAt", - size: 200, - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const syncedAt = row.getValue("syncedAt") as Date | null; - if (!syncedAt) { - return "-"; - } - - return ( - - ) - } - }, -] - -export const ConnectionsTable = ({ data }: { data: Connection[] }) => { - const [sorting, setSorting] = useState([]) - const [columnFilters, setColumnFilters] = useState([]) - const [columnVisibility, setColumnVisibility] = useState({}) - const [rowSelection, setRowSelection] = useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - numNoJobs, - } = useMemo(() => { - return { - numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length, - numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length, - numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length, - numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length, - numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length, - } - }, [data]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - onRowSelectionChange: setRowSelection, - columnResizeMode: 'onChange', - enableColumnResizing: false, - state: { - sorting, - columnFilters, - columnVisibility, - rowSelection, - }, - }) - - return ( -
-
- table.getColumn("name")?.setFilterValue(event.target.value)} - className="max-w-sm" - /> - - -
-
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ))} - - )) - ) : ( - - - No results. - - - )} - -
-
-
-
- {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx b/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx new file mode 100644 index 000000000..0c41c6b38 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx @@ -0,0 +1,203 @@ +"use client"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { getConnectionSyncJobLogs } from "@/features/workerApi/actions"; +import { cn, isServiceError } from "@/lib/utils"; +import type { JobLogEntry, JobLogLevel } from "@sourcebot/shared"; +import { Loader2, RefreshCw } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; + +const LOG_POLL_INTERVAL_MS = 1000; + +const JOB_LOG_LEVEL_CLASS_NAMES: Record = { + debug: "text-muted-foreground", + info: "text-blue-600 dark:text-blue-400", + warn: "text-amber-600 dark:text-amber-400", + error: "text-destructive", +}; + +const formatLogTimestamp = (timestamp: string | null) => { + if (!timestamp) { + return "—"; + } + return new Date(timestamp).toLocaleTimeString(); +}; + +type ConnectionSyncLogsDialogProps = { + connectionId: number; + connectionName: string; + jobId: string | null; + open: boolean; + onOpenChange: (open: boolean) => void; +}; + +export const ConnectionSyncLogsDialog = ({ + connectionId, + connectionName, + jobId, + open, + onOpenChange, +}: ConnectionSyncLogsDialogProps) => { + const [logs, setLogs] = useState([]); + const [hasLoaded, setHasLoaded] = useState(false); + const [isRefreshing, setIsRefreshing] = useState(false); + const [logsError, setLogsError] = useState(null); + const pollNowRef = useRef<() => void>(() => undefined); + + useEffect(() => { + if (!open || !jobId) { + return; + } + + let cursor = 0; + let isCancelled = false; + let isRequestInFlight = false; + + setLogs([]); + setHasLoaded(false); + setLogsError(null); + + const poll = async () => { + if (isRequestInFlight) { + return; + } + + isRequestInFlight = true; + setIsRefreshing(true); + + try { + const result = await getConnectionSyncJobLogs( + connectionId, + jobId, + cursor, + ); + if (isCancelled) { + return; + } + if (isServiceError(result)) { + setLogsError(result.message); + return; + } + + setLogsError(null); + setLogs((currentLogs) => ( + result.count < cursor + ? result.logs + : [...currentLogs, ...result.logs] + )); + cursor = result.count; + } catch { + if (!isCancelled) { + setLogsError("Failed to load logs for this sync."); + } + } finally { + if (!isCancelled) { + setHasLoaded(true); + setIsRefreshing(false); + } + isRequestInFlight = false; + } + }; + + const pollNow = () => { + void poll(); + }; + pollNowRef.current = pollNow; + + pollNow(); + const interval = window.setInterval(pollNow, LOG_POLL_INTERVAL_MS); + + return () => { + isCancelled = true; + window.clearInterval(interval); + if (pollNowRef.current === pollNow) { + pollNowRef.current = () => undefined; + } + }; + }, [connectionId, jobId, open]); + + const isInitialLoading = !hasLoaded && isRefreshing; + + return ( + + +
+ + {connectionName} sync logs + + Latest sync job {jobId}. Logs update automatically while this dialog is open. + + + +
+ +
+ {isInitialLoading ? ( +
+ + Loading logs… +
+ ) : logs.length > 0 ? ( + <> + {logsError && ( +
+ {logsError} +
+ )} +
+ {logs.map((entry, index) => ( +
+ + {formatLogTimestamp(entry.timestamp)} + + + {entry.level} + +

{entry.message}

+
+ ))} +
+ + ) : logsError ? ( +
+ {logsError} +
+ ) : ( +
+ No logs were recorded for this job. +
+ )} +
+
+
+ ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/connectionsList.tsx b/packages/web/src/app/(app)/settings/connections/connectionsList.tsx new file mode 100644 index 000000000..49acc679b --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/connectionsList.tsx @@ -0,0 +1,282 @@ +"use client"; + +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { useToast } from "@/components/hooks/use-toast"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Input } from "@/components/ui/input"; +import { syncConnection } from "@/features/workerApi/actions"; +import { cn, getCodeHostIcon, isServiceError } from "@/lib/utils"; +import { ConnectionType } from "@sourcebot/db"; +import { AlertCircle, CheckCircle2, CircleDashed, FileText, MoreHorizontal, RefreshCw, Search } from "lucide-react"; +import Image from "next/image"; +import { useRouter } from "next/navigation"; +import { useEffect, useMemo, useState } from "react"; +import { ConnectionSyncLogsDialog } from "./connectionSyncLogsDialog"; +import { WorkloadJob } from "@sourcebot/shared"; + +const SYNC_STATUS_POLL_INTERVAL_MS = 1000; + +export type ConnectionV2 = { + id: number; + name: string; + connectionType: ConnectionType; + syncedAt: Date | null; + currentJob: WorkloadJob<'connection'> | null; +}; + +const getConnectionStatus = (connection: ConnectionV2) => { + switch (connection.currentJob?.status) { + case "PENDING": + return { + key: "syncing", + label: "Queued", + icon: , + className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", + }; + case "IN_PROGRESS": + return { + key: "syncing", + label: "Syncing", + icon: , + className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", + }; + case "FAILED": + return { + key: "failed", + label: "Sync failed", + icon: , + className: "border-destructive/30 bg-destructive/10 text-destructive", + }; + case "COMPLETED": + return { + key: "healthy", + label: "Healthy", + icon: , + className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", + }; + default: + return connection.syncedAt + ? { + key: "healthy", + label: "Healthy", + icon: , + className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", + } + : { + key: "not-synced", + label: "Never synced", + icon: , + className: "border-border bg-muted text-muted-foreground", + }; + } +}; + +const ConnectionRow = ({ connection }: { connection: ConnectionV2 }) => { + const [isSubmitting, setIsSubmitting] = useState(false); + const [submittedJobId, setSubmittedJobId] = useState(null); + const [isLogsOpen, setIsLogsOpen] = useState(false); + const router = useRouter(); + const { toast } = useToast(); + const codeHostIcon = getCodeHostIcon(connection.connectionType); + const isRunning = connection.currentJob?.status === "PENDING" || + connection.currentJob?.status === "IN_PROGRESS"; + const isSubmittedJobSettled = connection.currentJob?.id === submittedJobId && + (connection.currentJob.status === "COMPLETED" || connection.currentJob.status === "FAILED"); + const isSyncRunning = isSubmitting || isRunning || (submittedJobId !== null && !isSubmittedJobSettled); + const status = isSyncRunning && !isRunning + ? { + key: "syncing", + label: "Syncing", + icon: , + className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", + } + : getConnectionStatus(connection); + + useEffect(() => { + if (!isSyncRunning) { + return; + } + + const interval = window.setInterval(() => { + router.refresh(); + }, SYNC_STATUS_POLL_INTERVAL_MS); + + return () => { + window.clearInterval(interval); + }; + }, [isSyncRunning, router]); + + const onSync = async () => { + setIsSubmitting(true); + try { + const result = await syncConnection(connection.id); + if (isServiceError(result)) { + toast({ + description: `❌ Failed to sync connection. ${result.message}`, + }); + return; + } + + setSubmittedJobId(result.jobId); + toast({ + description: `✅ Connection sync triggered. Job ID: ${result.jobId}`, + }); + router.refresh(); + } catch { + toast({ + description: "❌ Failed to sync connection.", + }); + } finally { + setIsSubmitting(false); + } + }; + + return ( + <> +
+
+
+ {`${connection.connectionType} +
+
+

{connection.name}

+

{connection.connectionType}

+
+
+ +
+

Current status

+ + {status.icon} + {status.label} + + {connection.currentJob?.status === "FAILED" && connection.currentJob.errorMessage && ( +

{connection.currentJob.errorMessage}

+ )} +
+ +
+

Last successful sync

+

+ {connection.syncedAt + ? + : Not yet synced} +

+
+ + + + + + + { + setIsLogsOpen(true); + }} + > + + View logs + + { + void onSync(); + }} + > + + {isSyncRunning ? "Sync in progress" : "Trigger sync"} + + + +
+ + + + ); +}; + +export const ConnectionsList = ({ data }: { data: ConnectionV2[] }) => { + const [query, setQuery] = useState(""); + const healthyCount = useMemo( + () => data.filter((connection) => getConnectionStatus(connection).key === "healthy").length, + [data], + ); + const filteredConnections = useMemo(() => { + const normalizedQuery = query.trim().toLowerCase(); + + return data.filter((connection) => ( + normalizedQuery.length === 0 || + connection.name.toLowerCase().includes(normalizedQuery) || + connection.connectionType.toLowerCase().includes(normalizedQuery) + )); + }, [data, query]); + + return ( +
+
+
+ + setQuery(event.target.value)} + placeholder="Search connections..." + className="pl-9" + /> +
+

+ {healthyCount} healthy · {data.length} total +

+
+ +
+ {filteredConnections.length > 0 ? filteredConnections.map((connection, index) => ( +
0 ? "border-t" : undefined}> + +
+ )) : ( +
+

{data.length === 0 ? "No code host connections" : "No matching connections"}

+

+ {data.length === 0 + ? "Add a connection to begin syncing repositories." + : "Try changing your search."} +

+
+ )} +
+ + {filteredConnections.length > 0 && ( +

+ Showing {filteredConnections.length} of {data.length} {data.length === 1 ? "connection" : "connections"} +

+ )} +
+ ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/page.tsx b/packages/web/src/app/(app)/settings/connections/page.tsx index fdce7b08f..a1d99cd38 100644 --- a/packages/web/src/app/(app)/settings/connections/page.tsx +++ b/packages/web/src/app/(app)/settings/connections/page.tsx @@ -1,77 +1,72 @@ -import { sew } from "@/middleware/sew"; import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; +import { sew } from "@/middleware/sew"; import { withAuth } from "@/middleware/withAuth"; +import { getBullMQClient } from "@/lib/bullmqClient"; import Link from "next/link"; -import { ConnectionsTable } from "./components/connectionsTable"; -import { ConnectionSyncJobStatus } from "@prisma/client"; +import { ConnectionsList } from "./connectionsList"; +import { CONNECTION_QUEUE } from "@sourcebot/shared"; const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; -export default async function ConnectionsPage() { - const _connections = await getConnectionsWithLatestJob(); - if (isServiceError(_connections)) { - throw new ServiceErrorException(_connections); +export default async function ConnectionsV2Page() { + const connections = await getConnectionsWithCurrentStatus(); + if (isServiceError(connections)) { + throw new ServiceErrorException(connections); } - // Sort connections so that first time syncs are at the top. - const connections = _connections - .map((connection) => ({ - ...connection, - isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0, - latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null, - })) - .sort((a, b) => { - if (a.isFirstTimeSync && !b.isFirstTimeSync) { - return -1; - } - if (!a.isFirstTimeSync && b.isFirstTimeSync) { - return 1; - } - return a.name.localeCompare(b.name); - }); - return (
-

Code Host Connections

-

Manage your connections to external code hosts. Learn more

+
+

Code Host Connections

+ + Prototype + +
+

+ Monitor and sync your external code hosts.{" "} + + Learn more + +

- ({ - id: connection.id, - name: connection.name, - connectionType: connection.connectionType, - syncedAt: connection.syncedAt, - latestJobStatus: connection.latestJobStatus, - isFirstTimeSync: connection.isFirstTimeSync, - }))} /> + +
- ) + ); } -const getConnectionsWithLatestJob = async () => sew(() => +const getConnectionsWithCurrentStatus = async () => sew(() => withAuth(async ({ prisma, org }) => { const connections = await prisma.connection.findMany({ where: { orgId: org.id, }, - include: { - _count: { - select: { - syncJobs: true, - } - }, - syncJobs: { - orderBy: { - createdAt: 'desc' - }, - take: 1 - }, + select: { + id: true, + name: true, + connectionType: true, + syncedAt: true, + latestSyncJobId: true, }, orderBy: { - name: 'asc' + name: 'asc', }, }); - return connections; - })); \ No newline at end of file + return Promise.all(connections.map(async ({ + latestSyncJobId, + ...connection + }) => { + const job = latestSyncJobId + ? await getBullMQClient().getJob(CONNECTION_QUEUE, latestSyncJobId) + : null; + + return { + ...connection, + currentJob: job + }; + })); + }) +); diff --git a/packages/web/src/app/(app)/settings/layout.tsx b/packages/web/src/app/(app)/settings/layout.tsx index 655ca28f5..e57e4180e 100644 --- a/packages/web/src/app/(app)/settings/layout.tsx +++ b/packages/web/src/app/(app)/settings/layout.tsx @@ -3,7 +3,6 @@ import { Metadata } from "next" import { redirect } from "next/navigation"; import { auth } from "@/auth"; import { isServiceError } from "@/lib/utils"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { OrgRole } from "@prisma/client"; @@ -48,10 +47,6 @@ export const getSidebarNavGroups = async () => numJoinRequests = requests.length; } - const connectionStats = await getConnectionStats(); - if (isServiceError(connectionStats)) { - throw new ServiceErrorException(connectionStats); - } const hasAskEntitlement = await hasEntitlement("ask"); const groups: NavGroup[] = [ @@ -119,8 +114,6 @@ export const getSidebarNavGroups = async () => { title: "Connections", href: `/settings/connections`, - hrefRegex: `/settings/connections(/[^/]+)?$`, - isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0, icon: "plug" as const, }, { diff --git a/packages/web/src/features/workerApi/actions.test.ts b/packages/web/src/features/workerApi/actions.test.ts deleted file mode 100644 index 0a949d983..000000000 --- a/packages/web/src/features/workerApi/actions.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { ConnectionSyncJobStatus, OrgRole } from '@sourcebot/db'; -import type { DataOf, QueueSpec } from '@sourcebot/shared'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -const mocks = vi.hoisted(() => ({ - connectionFindUnique: vi.fn(), - connectionSyncJobCreateMany: vi.fn(), - enqueue: vi.fn(), -})); - -vi.mock('@/middleware/sew', () => ({ - sew: (fn: () => Promise) => fn(), -})); - -vi.mock('@/middleware/withAuth', () => ({ - withAuth: (fn: (context: unknown) => Promise) => fn({ - org: { id: 7 }, - prisma: { - connection: { - findUnique: mocks.connectionFindUnique, - }, - connectionSyncJob: { - createMany: mocks.connectionSyncJobCreateMany, - }, - }, - role: 'OWNER', - }), - withOptionalAuth: vi.fn(), -})); - -vi.mock('@/middleware/withMinimumOrgRole', () => ({ - withMinimumOrgRole: ( - _role: OrgRole, - _minimumRole: OrgRole, - fn: () => Promise, - ) => fn(), -})); - -vi.mock('@/lib/jobProducer', () => ({ - getJobProducer: () => ({ - enqueue: mocks.enqueue, - }), -})); - -vi.mock('@sourcebot/shared', () => ({ - CONNECTION_QUEUE: { - name: 'connection', - dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - }, - }, - env: { - WORKER_API_URL: 'http://localhost:3060', - }, -})); - -import { syncConnection } from './actions'; - -describe('syncConnection', () => { - beforeEach(() => { - vi.clearAllMocks(); - mocks.connectionSyncJobCreateMany.mockResolvedValue({ count: 1 }); - mocks.enqueue.mockImplementation(async ( - spec: QueueSpec<'connection'>, - data: DataOf<'connection'>, - ) => { - const jobId = 'job-1'; - await spec.onEnqueued?.({ - data, - jobId, - attemptsMade: 0, - maxAttempts: spec.jobOptions.attempts, - }); - return jobId; - }); - }); - - test('enqueues an org-scoped connection sync and creates its pending job record', async () => { - mocks.connectionFindUnique.mockResolvedValue({ - id: 42, - orgId: 7, - }); - - const result = await syncConnection(42); - - expect(mocks.connectionFindUnique).toHaveBeenCalledWith({ - where: { - id: 42, - orgId: 7, - }, - select: { - id: true, - orgId: true, - }, - }); - expect(mocks.enqueue).toHaveBeenCalledWith( - expect.objectContaining({ - name: 'connection', - onEnqueued: expect.any(Function), - }), - { - connectionId: 42, - orgId: 7, - }, - ); - expect(mocks.connectionSyncJobCreateMany).toHaveBeenCalledWith({ - data: [{ - id: 'job-1', - connectionId: 42, - status: ConnectionSyncJobStatus.PENDING, - warningMessages: [], - }], - skipDuplicates: true, - }); - expect(result).toEqual({ jobId: 'job-1' }); - }); - - test('does not enqueue a missing connection', async () => { - mocks.connectionFindUnique.mockResolvedValue(null); - - const result = await syncConnection(42); - - expect(mocks.enqueue).not.toHaveBeenCalled(); - expect(result).toEqual(expect.objectContaining({ - statusCode: 404, - message: 'Connection not found', - })); - }); -}); diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index b0fa45ac5..54f0c7f24 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -4,10 +4,10 @@ import { sew } from "@/middleware/sew"; import { notFound, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; -import { ConnectionSyncJobStatus, OrgRole } from "@sourcebot/db"; +import { OrgRole } from "@sourcebot/db"; import { CONNECTION_QUEUE, env } from "@sourcebot/shared"; import z from "zod"; -import { getJobProducer } from "@/lib/jobProducer"; +import { getBullMQClient } from "@/lib/bullmqClient"; const WORKER_API_URL = env.WORKER_API_URL; @@ -29,7 +29,7 @@ export const syncConnection = async (connectionId: number) => sew(() => return notFound('Connection not found'); } - const jobId = await getJobProducer().enqueue(CONNECTION_QUEUE, { + const jobId = await getBullMQClient().enqueue(CONNECTION_QUEUE, { connectionId: connection.id, orgId: connection.orgId, }); @@ -39,6 +39,45 @@ export const syncConnection = async (connectionId: number) => sew(() => ) ); +export const getConnectionSyncJobLogs = async ( + connectionId: number, + jobId: string, + start = 0, +) => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const connection = await prisma.connection.findUnique({ + where: { + id: connectionId, + orgId: org.id, + }, + select: { + id: true, + }, + }); + + if (!connection) { + return notFound('Connection not found'); + } + + const client = getBullMQClient(); + const job = await client.getJob(CONNECTION_QUEUE, jobId); + if ( + !job || + job.data.connectionId !== connection.id || + job.data.orgId !== org.id + ) { + return notFound('Connection sync job not found'); + } + + return client.getJobLogs(CONNECTION_QUEUE, jobId, { + start: Number.isInteger(start) && start >= 0 ? start : 0, + ascending: true, + }); + }) + ) +); + export const indexRepo = async (repoId: number) => sew(() => withAuth(({ role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { diff --git a/packages/web/src/lib/bullmqClient.ts b/packages/web/src/lib/bullmqClient.ts new file mode 100644 index 000000000..ef5d9524f --- /dev/null +++ b/packages/web/src/lib/bullmqClient.ts @@ -0,0 +1,12 @@ +import 'server-only'; + +import { BullMQClient } from '@sourcebot/shared'; +import { getRedisClient } from './redis'; +import { __unsafePrisma } from '@/prisma'; + +let client: BullMQClient | undefined; + +export function getBullMQClient() { + client ??= new BullMQClient(getRedisClient(), __unsafePrisma); + return client; +} diff --git a/packages/web/src/lib/jobProducer.ts b/packages/web/src/lib/jobProducer.ts deleted file mode 100644 index 1d50ebe6d..000000000 --- a/packages/web/src/lib/jobProducer.ts +++ /dev/null @@ -1,11 +0,0 @@ -import 'server-only'; - -import { BullMQJobProducer } from '@sourcebot/shared'; -import { getRedisClient } from './redis'; - -let jobProducer: BullMQJobProducer | undefined; - -export function getJobProducer() { - jobProducer ??= new BullMQJobProducer(getRedisClient()); - return jobProducer; -} From c7108d668dc81ff3227c996f3f976961256080a8 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Thu, 30 Jul 2026 16:39:29 -0700 Subject: [PATCH 05/10] migrated repo indexing to workload --- packages/backend/package.json | 1 - packages/backend/src/api.ts | 163 +--- packages/backend/src/index.ts | 51 +- packages/backend/src/jobManager.test.ts | 163 +++- .../backend/src/jobManagerLifecycle.test.ts | 176 ---- .../src/reconciliationWorkload.test.ts | 114 ++- .../backend/src/reconciliationWorkload.ts | 52 +- packages/backend/src/repoIndexManager.test.ts | 912 ------------------ packages/backend/src/repoIndexManager.ts | 769 --------------- packages/backend/src/repoIndexWorkload.ts | 351 +++++++ packages/backend/src/types/redlock.d.ts | 95 -- packages/backend/src/utils.ts | 4 +- .../migration.sql | 22 + packages/db/prisma/schema.prisma | 33 +- packages/db/tools/scripts/inject-repo-data.ts | 22 +- packages/shared/src/bullmqClient.test.ts | 34 + packages/shared/src/index.server.ts | 3 +- packages/shared/src/queue.ts | 29 + packages/shared/src/types.ts | 9 - .../src/app/(app)/chat/chatLandingPage.tsx | 11 +- packages/web/src/app/(app)/repos/layout.tsx | 9 - .../search/components/searchLandingPage.tsx | 8 +- yarn.lock | 12 +- 23 files changed, 766 insertions(+), 2277 deletions(-) delete mode 100644 packages/backend/src/jobManagerLifecycle.test.ts delete mode 100644 packages/backend/src/repoIndexManager.test.ts delete mode 100644 packages/backend/src/repoIndexManager.ts create mode 100644 packages/backend/src/repoIndexWorkload.ts delete mode 100644 packages/backend/src/types/redlock.d.ts create mode 100644 packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql diff --git a/packages/backend/package.json b/packages/backend/package.json index 13640cef7..8d9f61aaf 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -52,7 +52,6 @@ "p-limit": "^7.2.0", "posthog-node": "^5.24.15", "prom-client": "^15.1.3", - "redlock": "5.0.0-beta.2", "simple-git": "^3.36.0", "zod": "^3.25.74" } diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 33f854ada..fd1d91ba5 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -1,18 +1,8 @@ -import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; -import { hasEntitlement } from './entitlements.js'; -import { createLogger, doesIdpSupportPermissionSyncing, env } from '@sourcebot/shared'; +import { createLogger, env } from '@sourcebot/shared'; import express, { Request, Response } from 'express'; import 'express-async-errors'; import * as http from "http"; -import { ConnectionManager } from './connectionManager.js'; -import { AccountPermissionSyncer } from './ee/accountPermissionSyncer.js'; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from './repoIndexManager.js'; -import { createGitHubRepoRecord } from './repoCompileUtils.js'; -import { isNotFound } from './errors.js'; -import { Octokit } from '@octokit/rest'; -import { SINGLE_TENANT_ORG_ID } from './constants.js'; -import z from 'zod'; const logger = createLogger('api'); @@ -22,13 +12,7 @@ const PORT = Number(workerApiUrl.port) || (workerApiUrl.protocol === "https:" ? export class Api { private server: http.Server; - constructor( - promClient: PromClient, - private prisma: PrismaClient, - private connectionManager: ConnectionManager, - private repoIndexManager: RepoIndexManager, - private accountPermissionSyncer: AccountPermissionSyncer, - ) { + constructor(promClient: PromClient) { const app = express(); app.use(express.json()); app.use(express.urlencoded({ extended: true })); @@ -40,154 +24,11 @@ export class Api { res.end(metrics); }); - app.post('/api/sync-connection', this.syncConnection.bind(this)); - app.post('/api/index-repo', this.indexRepo.bind(this)); - app.post('/api/trigger-account-permission-sync', this.triggerAccountPermissionSync.bind(this)); - app.post(`/api/experimental/add-github-repo`, this.experimental_addGithubRepo.bind(this)); - this.server = app.listen(PORT, () => { logger.debug(`API server is running on port ${PORT}`); }); } - private async syncConnection(req: Request, res: Response) { - const schema = z.object({ - connectionId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { connectionId } = parsed.data; - const connection = await this.prisma.connection.findUnique({ - where: { - id: connectionId, - } - }); - - if (!connection) { - res.status(404).json({ error: 'Connection not found' }); - return; - } - - const [jobId] = await this.connectionManager.createJobs([connection]); - - res.status(200).json({ jobId }); - } - - private async indexRepo(req: Request, res: Response) { - const schema = z.object({ - repoId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { repoId } = parsed.data; - const repo = await this.prisma.repo.findUnique({ - where: { id: repoId }, - }); - - if (!repo) { - res.status(404).json({ error: 'Repo not found' }); - return; - } - - const [jobId] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); - res.status(200).json({ jobId }); - } - - private async triggerAccountPermissionSync(req: Request, res: Response) { - if (env.PERMISSION_SYNC_ENABLED !== 'true' || !await hasEntitlement('permission-syncing')) { - res.status(403).json({ error: 'Permission syncing is not enabled.' }); - return; - } - - const schema = z.object({ - accountId: z.string(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { accountId } = parsed.data; - const account = await this.prisma.account.findUnique({ - where: { id: accountId }, - }); - - if (!account) { - res.status(404).json({ error: 'Account not found' }); - return; - } - - if (!doesIdpSupportPermissionSyncing(account.providerType)) { - res.status(400).json({ error: `Provider '${account.providerType}' does not support permission syncing.` }); - return; - } - - const jobId = await this.accountPermissionSyncer.schedulePermissionSyncForAccount(account); - res.status(200).json({ jobId }); - } - - private async experimental_addGithubRepo(req: Request, res: Response) { - const schema = z.object({ - owner: z.string(), - repo: z.string(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const octokit = new Octokit(); - let response; - try { - response = await octokit.rest.repos.get({ - owner: parsed.data.owner, - repo: parsed.data.repo, - }); - } catch (error) { - if (isNotFound(error)) { - res.status(404).json({ error: 'Repository not found on GitHub' }); - return; - } - throw error; - } - - const record = createGitHubRepoRecord({ - repo: response.data, - hostUrl: 'https://github.com', - isAutoCleanupDisabled: true, - }); - - const repo = await this.prisma.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: record.external_id, - external_codeHostUrl: record.external_codeHostUrl, - orgId: SINGLE_TENANT_ORG_ID, - } - }, - update: record, - create: record, - }); - - const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); - - res.status(200).json({ jobId, repoId: repo.id }); - } - public async dispose() { return new Promise((resolve, reject) => { this.server.close((err) => { diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 9efa68768..73db4fc3f 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -16,6 +16,8 @@ import { PromClient } from './promClient.js'; import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; import { connectionWorkload } from "./connectionWorkload.js"; +import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; +import { Api } from "./api.js"; const logger = createLogger('backend-entrypoint'); @@ -38,7 +40,6 @@ try { process.exit(1); } -const promClient = new PromClient(); const settings = await getConfigSettings(env.CONFIG_PATH); @@ -46,51 +47,27 @@ if (await hasEntitlement('github-app')) { await GithubAppManager.getInstance().init(prisma); } -// const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); -// const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); -// const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); -// const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); -// const auditLogPruner = new AuditLogPruner(prisma); -// const attachmentPruner = new AttachmentPruner(prisma); - -// connectionManager.startScheduler(); -// await repoIndexManager.startScheduler(); -// auditLogPruner.startScheduler(); -// attachmentPruner.startScheduler(); - -// if (env.PERMISSION_SYNC_ENABLED === 'true' && !await hasEntitlement('permission-syncing')) { -// logger.warn('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); -// } -// else if (env.PERMISSION_SYNC_ENABLED === 'true' && await hasEntitlement('permission-syncing')) { -// if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { -// await repoPermissionSyncer.startScheduler(); -// } -// await accountPermissionSyncer.startScheduler(); -// } - -// const api = new Api( -// promClient, -// prisma, -// connectionManager, -// repoIndexManager, -// accountPermissionSyncer, -// ); +const promClient = new PromClient(); +const api = new Api(promClient); logger.info('Worker started.'); -// Background jobs run through the JobManager (BullMQ/Redis as the source of truth). Phase 0 -// wires the framework here in place of the old per-manager pollers; the real workloads -// (repo-index, connection-sync, permission syncers) are ported onto it in subsequent phases. const jobManager = new BullMQJobManager(redis); const reconciliationWorkload = createReconciliationWorkload({ db: prisma, settings, }); +const repoIndexWorkload = createRepoIndexWorkload({ + db: prisma, + settings, +}); jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); +jobManager.register(repoIndexWorkload); +await cleanupOrphanedRepoResources(prisma); await jobManager.start(); const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); @@ -110,18 +87,12 @@ const listenToShutdownSignals = () => { logger.info(`Received ${signal}, cleaning up...`); - // await repoIndexManager.dispose() - // await connectionManager.dispose() - // await repoPermissionSyncer.dispose() - // await accountPermissionSyncer.dispose() - // await auditLogPruner.dispose() - // await attachmentPruner.dispose() await configManager.dispose() await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); - // await api.dispose(); + await api.dispose(); await shutdownPosthog(); logger.info('All workers shut down gracefully'); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 5f01538a5..894efdade 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -1,4 +1,23 @@ -import { describe, expect, test, vi } from 'vitest'; +import { Redis } from 'ioredis'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { ProcessContext, Workload } from './types.js'; + +const mocks = vi.hoisted(() => ({ + enqueue: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + jobLogger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + flush: vi.fn(), + }, + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, +})); // The module under test creates a logger at import time; stub it so importing pure helpers // has no side effects (mirrors repoIndexManager.test.ts). @@ -9,6 +28,15 @@ vi.mock('@sourcebot/shared', () => ({ error: vi.fn(), debug: vi.fn(), })), + createBullMQJobLogger: vi.fn(() => mocks.jobLogger), + BullMQClient: class { + enqueue = mocks.enqueue; + close = mocks.producerClose; + getQueue = vi.fn(() => ({ + getJobCounts: vi.fn(), + upsertJobScheduler: vi.fn(), + })); + }, })); // Mock the constants module directly so its env-derived cache-dir paths don't load. @@ -16,7 +44,28 @@ vi.mock('./constants.js', () => ({ WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, })); -import { normalizeJobState, parseDuration } from './jobManager.js'; +vi.mock('@sentry/node', () => ({ + captureException: vi.fn(), +})); + +vi.mock('bullmq', () => ({ + Worker: class { + private readonly record: (typeof mocks.workers)[number]; + + constructor(_name: string, processor: (job: unknown) => Promise) { + this.record = { processor, handlers: new Map() }; + mocks.workers.push(this.record); + } + + on(event: string, handler: (...args: unknown[]) => void) { + this.record.handlers.set(event, handler); + } + + close = mocks.workerClose; + }, +})); + +import { BullMQJobManager, normalizeJobState, parseDuration } from './jobManager.js'; describe('parseDuration', () => { test.each([ @@ -61,3 +110,113 @@ describe('normalizeJobState', () => { }); }); +const createWorkload = ( + overrides: Partial> = {}, +): Workload<'connection', { repoCount: number }> => ({ + queueSpec: { + name: 'connection', + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, + concurrency: 2, + process: vi.fn(async () => ({ repoCount: 3 })), + ...overrides, +}); + +const data = { connectionId: 42, orgId: 1 }; +const job = { + id: 'job-1', + data, + attemptsMade: 2, + opts: { attempts: 2 }, + log: vi.fn(), + updateProgress: vi.fn(), +}; + +describe('BullMQJobManager lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.workers.length = 0; + mocks.enqueue.mockResolvedValue('job-1'); + }); + + test('delegates enqueueing to BullMQClient and returns its job id', async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + const result = await manager.trigger('connection', data); + + expect(result).toBe('job-1'); + expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); + }); + + test('calls onStarted before processing and onCompleted after completion', async () => { + const calls: string[] = []; + const workload = createWorkload({ + onStarted: vi.fn(async () => { calls.push('started'); }), + process: vi.fn(async () => { + calls.push('processed'); + return { repoCount: 3 }; + }), + onCompleted: vi.fn(async () => { calls.push('completed'); }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + expect(calls).toEqual(['started', 'processed']); + + mocks.workers[0].handlers.get('completed')?.(job, result); + await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); + expect(workload.onCompleted).toHaveBeenCalledWith( + expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), + { repoCount: 3 }, + ); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test('provides the structured job logger to the workload processor', async () => { + const process = vi.fn(async (context: ProcessContext<'connection'>) => { + context.logger.info('Processing connection'); + return { repoCount: 3 }; + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload({ process })); + await manager.start(); + + await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + + expect(process).toHaveBeenCalledWith(expect.objectContaining({ + logger: mocks.jobLogger, + })); + expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test('reports lifecycle metadata after terminal failure', async () => { + const onTerminalFailure = vi.fn(); + const workload = createWorkload({ onTerminalFailure }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const error = new Error('failed'); + mocks.workers[0].handlers.get('failed')?.(job, error); + + await vi.waitFor(() => { + expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ + data, + jobId: 'job-1', + attemptsMade: 2, + maxAttempts: 2, + }), error); + }); + }); +}); diff --git a/packages/backend/src/jobManagerLifecycle.test.ts b/packages/backend/src/jobManagerLifecycle.test.ts deleted file mode 100644 index 86a1120f4..000000000 --- a/packages/backend/src/jobManagerLifecycle.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { ProcessContext, Workload } from './types.js'; - -const mocks = vi.hoisted(() => ({ - enqueue: vi.fn(), - producerClose: vi.fn(), - workerClose: vi.fn(), - jobLogger: { - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - flush: vi.fn(), - }, - workers: [] as Array<{ - processor: (job: unknown) => Promise; - handlers: Map void>; - }>, -})); - -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('@sourcebot/shared', () => ({ - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - createBullMQJobLogger: vi.fn(() => mocks.jobLogger), - BullMQClient: class { - enqueue = mocks.enqueue; - close = mocks.producerClose; - queue = vi.fn(() => ({ - getJobCounts: vi.fn(), - upsertJobScheduler: vi.fn(), - })); - }, -})); - -vi.mock('./constants.js', () => ({ - WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, -})); - -vi.mock('bullmq', () => ({ - Worker: class { - private readonly record: (typeof mocks.workers)[number]; - - constructor(_name: string, processor: (job: unknown) => Promise) { - this.record = { processor, handlers: new Map() }; - mocks.workers.push(this.record); - } - - on(event: string, handler: (...args: unknown[]) => void) { - this.record.handlers.set(event, handler); - } - - close = mocks.workerClose; - }, -})); - -import { BullMQJobManager } from './jobManager.js'; - -const createWorkload = ( - overrides: Partial> = {}, -): Workload<'connection', { repoCount: number }> => ({ - queueSpec: { - name: 'connection', - dedupKey: ({ connectionId }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: 500, - }, - }, - concurrency: 2, - process: vi.fn(async () => ({ repoCount: 3 })), - ...overrides, -}); - -const data = { connectionId: 42, orgId: 1 }; -const job = { - id: 'job-1', - data, - attemptsMade: 2, - opts: { attempts: 2 }, - log: vi.fn(), - updateProgress: vi.fn(), -}; - -describe('BullMQJobManager lifecycle', () => { - beforeEach(() => { - vi.clearAllMocks(); - mocks.workers.length = 0; - mocks.enqueue.mockResolvedValue('job-1'); - }); - - test('delegates enqueueing to BullMQClient and returns its job id', async () => { - const manager = new BullMQJobManager({} as Redis); - const workload = createWorkload(); - manager.register(workload); - - const result = await manager.trigger('connection', data); - - expect(result).toBe('job-1'); - expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); - }); - - test('calls onStarted before processing and onCompleted after completion', async () => { - const calls: string[] = []; - const workload = createWorkload({ - onStarted: vi.fn(async () => { calls.push('started'); }), - process: vi.fn(async () => { - calls.push('processed'); - return { repoCount: 3 }; - }), - onCompleted: vi.fn(async () => { calls.push('completed'); }), - }); - const manager = new BullMQJobManager({} as Redis); - manager.register(workload); - await manager.start(); - - const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - expect(calls).toEqual(['started', 'processed']); - - mocks.workers[0].handlers.get('completed')?.(job, result); - await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); - expect(workload.onCompleted).toHaveBeenCalledWith( - expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), - { repoCount: 3 }, - ); - expect(mocks.jobLogger.flush).toHaveBeenCalled(); - }); - - test('provides the structured job logger to the workload processor', async () => { - const process = vi.fn(async (context: ProcessContext<'connection'>) => { - context.logger.info('Processing connection'); - return { repoCount: 3 }; - }); - const manager = new BullMQJobManager({} as Redis); - manager.register(createWorkload({ process })); - await manager.start(); - - await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - - expect(process).toHaveBeenCalledWith(expect.objectContaining({ - logger: mocks.jobLogger, - })); - expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); - expect(mocks.jobLogger.flush).toHaveBeenCalled(); - }); - - test('reports lifecycle metadata after terminal failure', async () => { - const onTerminalFailure = vi.fn(); - const workload = createWorkload({ onTerminalFailure }); - const manager = new BullMQJobManager({} as Redis); - manager.register(workload); - await manager.start(); - - const error = new Error('failed'); - mocks.workers[0].handlers.get('failed')?.(job, error); - - await vi.waitFor(() => { - expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ - data, - jobId: 'job-1', - attemptsMade: 2, - maxAttempts: 2, - }), error); - }); - }); -}); diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index 3aea21082..949af1c50 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -1,4 +1,5 @@ import type { PrismaClient } from '@sourcebot/db'; +import type { JobLogger } from '@sourcebot/shared'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; vi.mock('@sourcebot/shared', () => ({ @@ -11,25 +12,41 @@ vi.mock('@sourcebot/shared', () => ({ keepLogs: 500, }, }, - createLogger: vi.fn(() => ({ - debug: vi.fn(), - })), })); import { createReconciliationWorkload } from './reconciliationWorkload.js'; +const settings = { + resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, + reindexIntervalMs: 60 * 60 * 1000, + repoGarbageCollectionGracePeriodMs: 10 * 1000, +}; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + describe('reconciliationWorkload', () => { - const findMany = vi.fn(); + const connectionFindMany = vi.fn(); + const repoFindMany = vi.fn(); const db = { connection: { - findMany, + findMany: connectionFindMany, + }, + repo: { + findMany: repoFindMany, }, } as unknown as PrismaClient; beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); - findMany.mockReset(); + connectionFindMany.mockReset().mockResolvedValue([]); + repoFindMany.mockReset().mockResolvedValue([]); }); afterEach(() => { @@ -39,9 +56,7 @@ describe('reconciliationWorkload', () => { test('runs every 15 minutes on the reconciliation queue', () => { const workload = createReconciliationWorkload({ db, - settings: { - resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, - }, + settings, }); expect(workload.queueSpec.name).toBe('reconciliation'); @@ -50,16 +65,14 @@ describe('reconciliationWorkload', () => { }); test('triggers connection syncs for connections that are due', async () => { - findMany.mockResolvedValue([ + connectionFindMany.mockResolvedValue([ { id: 42, orgId: 1 }, { id: 84, orgId: 2 }, ]); const trigger = vi.fn().mockResolvedValue('job-id'); const workload = createReconciliationWorkload({ db, - settings: { - resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, - }, + settings, }); await workload.process({ @@ -67,19 +80,14 @@ describe('reconciliationWorkload', () => { jobId: 'reconciliation-job', attemptsMade: 0, maxAttempts: 2, + prisma: db, signal: new AbortController().signal, - logger: { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - flush: vi.fn(), - }, + logger, updateProgress: vi.fn(), trigger, }); - expect(findMany).toHaveBeenCalledWith({ + expect(connectionFindMany).toHaveBeenCalledWith({ where: { OR: [ { syncedAt: null }, @@ -101,4 +109,68 @@ describe('reconciliationWorkload', () => { orgId: 2, }); }); + + test('submits all due repos for indexing', async () => { + repoFindMany + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { id: 1 }, + { id: 2 }, + { id: 3 }, + { id: 4 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(trigger).toHaveBeenCalledTimes(4); + for (const repoId of [1, 2, 3, 4]) { + expect(trigger).toHaveBeenCalledWith('repo-index', { + repoId, + type: 'INDEX', + }); + } + }); + + test('schedules orphaned repos for cleanup', async () => { + repoFindMany + .mockResolvedValueOnce([{ id: 42 }]) + .mockResolvedValueOnce([]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(trigger).toHaveBeenCalledWith('repo-index', { + repoId: 42, + type: 'CLEANUP', + }); + }); }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index 73209b26c..e36e7c975 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -4,7 +4,7 @@ import { Settings, Workload } from "./types.js"; interface ReconciliationWorkloadDependencies { db: PrismaClient; - settings: Pick; + settings: Settings; } export const createReconciliationWorkload = ({ @@ -15,12 +15,12 @@ export const createReconciliationWorkload = ({ schedule: { every: '15m' }, queueSpec: RECONCILIATION_QUEUE, process: async ({ logger, trigger }) => { - const thresholdDate = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); const connections = await db.connection.findMany({ where: { OR: [ { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, + { syncedAt: { lt: connectionThreshold } }, ], }, select: { @@ -36,5 +36,51 @@ export const createReconciliationWorkload = ({ orgId: connection.orgId, }); })); + + const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); + const reposToCleanup = await db.repo.findMany({ + where: { + connections: { + none: {}, + }, + isAutoCleanupDisabled: false, + OR: [ + { indexedAt: null }, + { indexedAt: { lt: cleanupThreshold } }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(reposToCleanup.map(async ({ id }) => { + logger.debug(`Scheduling cleanup for repo ${id}`); + await trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }); + })); + + const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); + const reposToIndex = await db.repo.findMany({ + where: { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: indexThreshold } }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(reposToIndex.map(async ({ id }) => { + logger.debug(`Scheduling index for repo ${id}`); + await trigger('repo-index', { + repoId: id, + type: 'INDEX', + }); + })); }, }); diff --git a/packages/backend/src/repoIndexManager.test.ts b/packages/backend/src/repoIndexManager.test.ts deleted file mode 100644 index 684f1a826..000000000 --- a/packages/backend/src/repoIndexManager.test.ts +++ /dev/null @@ -1,912 +0,0 @@ -import type { PrismaClient, Repo } from '@sourcebot/db'; -import { RepoIndexingJobStatus, RepoIndexingJobType } from '@sourcebot/db'; -import type { Job } from 'bullmq'; -import type { Redis } from 'ioredis'; -import { afterEach, beforeEach, describe, expect, Mock, test, vi } from 'vitest'; -import type { RepoWithConnections, Settings } from './types.js'; - -// Mock modules before importing the class under test -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('@sourcebot/shared', () => ({ - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - env: { - DATA_CACHE_DIR: 'test-data', - REDIS_REMOVE_ON_COMPLETE: true, - REDIS_REMOVE_ON_FAIL: true, - }, - getRepoPath: vi.fn((repo: Repo) => ({ - path: `/test-data/repos/${repo.id}`, - isReadOnly: false, - })), - repoMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, - repoIndexingJobMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, -})); - -vi.mock('./constants.js', () => ({ - WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, - INDEX_CACHE_DIR: 'test-data/index', -})); - -vi.mock('./git.js', () => ({ - cloneRepository: vi.fn(), - fetchRepository: vi.fn(), - getBranches: vi.fn().mockResolvedValue([]), - getTags: vi.fn().mockResolvedValue([]), - getLocalDefaultBranch: vi.fn().mockResolvedValue('main'), - getCommitHashForRefName: vi.fn().mockResolvedValue('abc123'), - getLatestCommitTimestamp: vi.fn().mockResolvedValue(new Date()), - isPathAValidGitRepoRoot: vi.fn().mockResolvedValue(true), - isRepoEmpty: vi.fn().mockResolvedValue(false), - unsetGitConfig: vi.fn(), - upsertGitConfig: vi.fn(), - writeCommitGraph: vi.fn(), -})); - -vi.mock('./zoekt.js', () => ({ - indexGitRepository: vi.fn().mockResolvedValue({ stdout: '', stderr: '' }), -})); - -vi.mock('./posthog.js', () => ({ - captureEvent: vi.fn(), -})); - -vi.mock('./utils.js', () => ({ - getAuthCredentialsForRepo: vi.fn().mockResolvedValue(null), - getShardPrefix: vi.fn((orgId: number, repoId: number) => `${orgId}_${repoId}`), - measure: vi.fn(async (cb: () => Promise) => { - const data = await cb(); - return { data, durationMs: 100 }; - }), - setIntervalAsync: vi.fn((cb: () => void, _interval: number) => { - // Return a mock interval ID - return { unref: vi.fn() } as unknown as NodeJS.Timeout; - }), -})); - -vi.mock('fs', () => ({ - existsSync: vi.fn().mockReturnValue(false), -})); - -vi.mock('fs/promises', () => ({ - rm: vi.fn(), - readdir: vi.fn().mockResolvedValue([]), -})); - -// Mock BullMQ -const mockQueueAdd = vi.fn().mockResolvedValue(undefined); -const mockQueueClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerOn = vi.fn(); - -vi.mock('bullmq', () => ({ - Queue: vi.fn().mockImplementation(function () { - return { - add: mockQueueAdd, - close: mockQueueClose, - }; - }), - Worker: vi.fn().mockImplementation(function (_name: string, processor: unknown) { - return { - on: mockWorkerOn, - close: mockWorkerClose, - processJob: processor, - }; - }), - DelayedError: class DelayedError extends Error { - constructor(message: string) { - super(message); - this.name = 'DelayedError'; - } - }, -})); - -// Mock Redlock -const mockRedlockUsing = vi.fn(); -vi.mock('redlock', () => ({ - default: vi.fn().mockImplementation(function () { - return { - using: mockRedlockUsing, - }; - }), - ExecutionError: class ExecutionError extends Error { - constructor(message: string) { - super(message); - this.name = 'ExecutionError'; - } - }, -})); - -// Import after mocks are set up -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { ExecutionError } from 'redlock'; -import { - cloneRepository, - fetchRepository, - getBranches, - getTags, - isPathAValidGitRepoRoot, -} from './git.js'; -import { RepoIndexManager } from './repoIndexManager.js'; -import { indexGitRepository } from './zoekt.js'; - -// Helper to create mock Prisma client -const createMockPrisma = () => { - return { - repo: { - findMany: vi.fn().mockResolvedValue([]), - update: vi.fn(), - delete: vi.fn(), - }, - repoIndexingJob: { - createManyAndReturn: vi.fn().mockResolvedValue([]), - findUniqueOrThrow: vi.fn().mockResolvedValue({ status: RepoIndexingJobStatus.PENDING }), - update: vi.fn(), - }, - } as unknown as PrismaClient; -}; - -// Helper to create mock Redis -const createMockRedis = () => { - return {} as Redis; -}; - -// Helper to create mock Settings -const createMockSettings = (): Settings => ({ - maxFileSize: 2 * 1024 * 1024, - maxTrigramCount: 20000, - reindexIntervalMs: 1000 * 60 * 60, - resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, - resyncConnectionPollingIntervalMs: 1000 * 1, - reindexRepoPollingIntervalMs: 1000 * 1, - maxConnectionSyncJobConcurrency: 8, - maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, - repoGarbageCollectionGracePeriodMs: 10 * 1000, - repoIndexTimeoutMs: 1000 * 60 * 60 * 2, - enablePublicAccess: false, - experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - maxAccountPermissionSyncJobConcurrency: 8, - maxRepoPermissionSyncJobConcurrency: 8, -}); - -// Helper to create mock PromClient -const createMockPromClient = () => ({ - pendingRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - activeRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - repoIndexJobSuccessTotal: { inc: vi.fn() }, - repoIndexJobFailTotal: { inc: vi.fn() }, -}); - -// Helper to create a mock repo -const createMockRepo = (overrides: Partial = {}): Repo => ({ - id: 1, - name: 'test-repo', - cloneUrl: 'https://github.com/test/repo.git', - orgId: 1, - indexedAt: null, - indexedCommitHash: null, - defaultBranch: 'main', - metadata: {}, - repoIndexingStatus: 'CREATED', - latestIndexingJobStatus: null, - latestConnectionSyncJobStatus: null, - external_id: 'test-external-id', - external_codeHostType: 'github', - external_codeHostUrl: 'https://github.com', - pushedAt: null, - createdAt: new Date(), - updatedAt: new Date(), - isFork: false, - isArchived: false, - isAutoCleanupDisabled: false, - ...overrides, -} as Repo); - -// Helper to create a mock repoWithConnections -const createMockRepoWithConnections = (overrides: Partial = {}): RepoWithConnections => ({ - ...createMockRepo(), - connections: [], - ...overrides, -}); - -describe('RepoIndexManager', () => { - let mockPrisma: PrismaClient; - let mockRedis: Redis; - let mockSettings: Settings; - let mockPromClient: ReturnType; - let manager: RepoIndexManager; - - beforeEach(() => { - vi.clearAllMocks(); - mockPrisma = createMockPrisma(); - mockRedis = createMockRedis(); - mockSettings = createMockSettings(); - mockPromClient = createMockPromClient(); - - // Default redlock behavior - execute the callback immediately - mockRedlockUsing.mockImplementation(async (_keys: string[], _ttl: number, cb: (signal: AbortSignal) => Promise) => { - const signal = new AbortController().signal; - return cb(signal); - }); - }); - - afterEach(async () => { - if (manager) { - await manager.dispose(); - } - }); - - describe('Job Processing - Success', () => { - test('clones new repository when directory does not exist', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // Set up mocks for job processing - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - // Simulate processing a job - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - // Get the worker processor callback - const { Worker } = await import('bullmq'); - const workerCalls = (Worker as unknown as Mock).mock.calls; - expect(workerCalls.length).toBeGreaterThan(0); - const processor = workerCalls[0][1]; - - // Execute the processor - await processor(mockJob); - - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('deletes directory and performs fresh clone when path exists but is not a valid git repo root', async () => { - const repo = createMockRepoWithConnections(); - // Path exists initially but after rm is called, it no longer exists - // First two calls return true (first check + check before delete), then false (after deletion) - (existsSync as Mock) - .mockReturnValueOnce(true) // First existsSync check - path exists - .mockReturnValueOnce(false); // Second existsSync check - path deleted, trigger clone - (isPathAValidGitRepoRoot as Mock).mockResolvedValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the invalid directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should perform a fresh clone after deletion - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('fetches existing repository when directory exists', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - (fetchRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(fetchRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('invokes zoekt-git-index with correct arguments', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - expect.arrayContaining(['refs/heads/main']), - expect.any(Object) - ); - }); - - test('keeps default branch and truncates to the first 63 matching tags', async () => { - const newestTagsFirst = Array.from( - { length: 70 }, - (_, index) => `v${70 - index}.0.0`, - ); - const repo = createMockRepoWithConnections({ - metadata: { - tags: ['**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getTags as Mock).mockResolvedValue(newestTagsFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - [ - 'refs/heads/main', - ...newestTagsFirst - .slice(0, 63) - .map((tag) => `refs/tags/${tag}`), - ], - expect.any(Object) - ); - }); - - test('de-duplicates the default branch before truncating matching branches', async () => { - const newestBranchesFirst = [ - 'feature/newest', - 'main', - ...Array.from( - { length: 68 }, - (_, index) => `feature/${68 - index}`, - ), - ]; - const repo = createMockRepoWithConnections({ - metadata: { - branches: ['main', 'feature/**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getBranches as Mock).mockResolvedValue(newestBranchesFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - const revisions = (indexGitRepository as Mock).mock.calls.at(-1)?.[2] as string[]; - - expect(revisions).toHaveLength(64); - expect(revisions.filter((revision) => revision === 'refs/heads/main')).toHaveLength(1); - expect(revisions[0]).toBe('refs/heads/main'); - expect(revisions).toContain('refs/heads/feature/newest'); - expect(revisions).not.toContain('refs/heads/feature/6'); - }); - - test('updates repo.indexedAt and indexedCommitHash on completion', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // The onJobCompleted handler reads the job via findUniqueOrThrow, then marks it - // COMPLETED and updates the repo (indexedAt, etc.) in a single repoIndexingJob.update - // with a nested repo update. - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - // Get the onCompleted handler - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - expect(onCompletedHandler).toBeDefined(); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - // The job status and indexedAt must be written together (single transaction) to - // close the race where the scheduler sees a completed job but a stale indexedAt. - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - completedAt: expect.any(Date), - repo: { - update: expect.objectContaining({ - indexedAt: expect.any(Date), - indexedCommitHash: 'abc123', - }), - }, - }), - }) - ); - }); - }); - - describe('Job Processing - Failure', () => { - test('marks job as FAILED when git clone throws', async () => { - const repo = createMockRepoWithConnections(); - const cloneError = new Error('Clone failed: authentication error'); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockRejectedValue(cloneError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - // Get the onFailed handler - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - expect(onFailedHandler).toBeDefined(); - - await onFailedHandler(mockJob, cloneError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: cloneError.message, - }), - }) - ); - - expect(mockPromClient.repoIndexJobFailTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'index', - }); - }); - - test('marks job as FAILED when zoekt-git-index fails', async () => { - const repo = createMockRepoWithConnections(); - const indexError = new Error('zoekt-git-index: failed to index'); - (existsSync as Mock).mockReturnValue(true); - (indexGitRepository as Mock).mockRejectedValue(indexError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - await onFailedHandler(mockJob, indexError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: indexError.message, - }), - }) - ); - }); - }); - - describe('Concurrency Control', () => { - test('prevents concurrent jobs for the same repo via redlock', async () => { - const repo = createMockRepoWithConnections(); - - // Simulate lock acquisition failure - mockRedlockUsing.mockRejectedValue(new ExecutionError('Lock already held')); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn().mockResolvedValue(undefined), - token: 'test-token', - } as unknown as Job; - - const { Worker, DelayedError } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - - // The processor should throw a DelayedError when lock cannot be acquired - await expect(processor(mockJob)).rejects.toThrow('locked'); - - // Verify moveToDelayed was called to retry later - expect(mockJob.moveToDelayed).toHaveBeenCalled(); - }); - }); - - describe('Cleanup Jobs', () => { - test('deletes repo directory and index shards', async () => { - const repo = createMockRepoWithConnections({ id: 5, orgId: 2 }); - (existsSync as Mock).mockReturnValue(true); - (readdir as Mock).mockResolvedValue(['2_5_v1.zoekt', '2_5_v2.zoekt', 'other_file.zoekt']); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repo, - }); - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the repo directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should delete shard files matching the prefix - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v1.zoekt'), - { force: true } - ); - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v2.zoekt'), - { force: true } - ); - }); - - test('removes repo from database after cleanup', async () => { - const repo = createMockRepoWithConnections({ id: 3 }); - (existsSync as Mock).mockReturnValue(false); - (readdir as Mock).mockResolvedValue([]); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - (mockPrisma.repo.delete as Mock).mockResolvedValue(repo); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repo.delete).toHaveBeenCalledWith({ - where: { id: repo.id }, - }); - - expect(mockPromClient.repoIndexJobSuccessTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'cleanup', - }); - }); - }); - - describe('latestIndexingJobStatus Updates', () => { - test('sets latestIndexingJobStatus to IN_PROGRESS when job starts', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - // Ensure indexGitRepository resolves for this test - (indexGitRepository as Mock).mockResolvedValue({ stdout: '', stderr: '' }); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Verify the first update call sets latestIndexingJobStatus to IN_PROGRESS - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - }, - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to COMPLETED when job succeeds', async () => { - const repo = createMockRepoWithConnections(); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - repo: { - update: expect.objectContaining({ - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - }), - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to FAILED when job fails', async () => { - const repo = createMockRepoWithConnections(); - const error = new Error('Job processing failed'); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - await onFailedHandler(mockJob, error); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - }, - }, - }), - }) - ); - }); - }); -}); diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts deleted file mode 100644 index aea1291dc..000000000 --- a/packages/backend/src/repoIndexManager.ts +++ /dev/null @@ -1,769 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; -import { createLogger, env, getRepoPath, Logger, getRepoIdFromPath, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { Redis } from 'ioredis'; -import micromatch from 'micromatch'; -import Redlock, { ExecutionError } from 'redlock'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js'; -import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; -import { captureEvent } from './posthog.js'; -import { PromClient } from './promClient.js'; -import { RepoWithConnections, Settings } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure, setIntervalAsync } from './utils.js'; -import { cleanupTempShards, indexGitRepository } from './zoekt.js'; - -const LOG_TAG = 'repo-index-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'repo-index-queue'; - -type JobPayload = { - type: 'INDEX' | 'CLEANUP'; - jobId: string; - repoId: number; - repoName: string; -}; - -// Lock TTL with auto-extension - minimizes dead lock time after crashes -const LOCK_TTL_MS = 60 * 1000; // 1 minute -const LOCK_PREFIX = `bullmq:${QUEUE_NAME}:lock:`; - -// Delay before retrying a job when the group lock cannot be acquired -const LOCK_RETRY_DELAY_MS = 5000; - -/** - * Manages the lifecycle of repository data on disk, including git working copies - * and search index shards. Handles both indexing operations (cloning/fetching repos - * and building search indexes) and cleanup operations (removing orphaned repos and - * their associated data). - * - * Uses a job queue system to process indexing and cleanup tasks asynchronously, - * with configurable concurrency limits and retry logic. Automatically schedules - * re-indexing of repos based on configured intervals and manages garbage collection - * of repos that are no longer connected to any source. - */ -export class RepoIndexManager { - private interval?: NodeJS.Timeout; - private queue: Queue; - private worker: Worker; - private redlock: Redlock; - private abortController: AbortController; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.redlock = new Redlock([redis], { - retryCount: 0, // Don't retry - we'll delay the job instead - automaticExtensionThreshold: LOCK_TTL_MS / 2, // Extend when 50% of TTL remains - }); - - this.worker = new Worker( - QUEUE_NAME, - this.processJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxRepoIndexingJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Index syncer worker error:`, error); - }); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - // Cleanup any orphaned disk resources on startup - await this.cleanupOrphanedDiskResources(); - this.interval = setIntervalAsync(async () => { - await this.scheduleIndexJobs(); - await this.scheduleCleanupJobs(); - }, this.settings.reindexRepoPollingIntervalMs); - } - - private async scheduleIndexJobs() { - const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToIndex = await this.db.repo.findMany({ - where: { - AND: [ - { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.INDEX, - }, - { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: RepoIndexingJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - ] - } - } - } - } - ], - }, - }); - - if (reposToIndex.length > 0) { - await this.createJobs(reposToIndex, RepoIndexingJobType.INDEX); - } - } - - private async scheduleCleanupJobs() { - const gcGracePeriodMs = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToCleanup = await this.db.repo.findMany({ - where: { - connections: { - none: {} - }, - isAutoCleanupDisabled: false, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: gcGracePeriodMs } }, - ], - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.CLEANUP, - }, - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - } - } - } - } - }); - - if (reposToCleanup.length > 0) { - await this.createJobs(reposToCleanup, RepoIndexingJobType.CLEANUP); - } - } - - public async createJobs(repos: Repo[], type: RepoIndexingJobType) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoIndexingJob.createManyAndReturn({ - data: repos.map(repo => ({ - type, - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - for (const job of jobs) { - await this.queue.add( - 'repo-index-job', - { - jobId: job.id, - type, - repoName: job.repo.name, - repoId: job.repo.id, - }, - { jobId: job.id } - ); - - const jobTypeLabel = getJobTypePrometheusLabel(type); - this.promClient.pendingRepoIndexJobs.inc({ repo: job.repo.name, type: jobTypeLabel }); - } - - return jobs.map(job => job.id); - } - - private async processJob(job: Job): Promise { - const groupId = `repo:${job.data.repoId}`; - const lockKey = `${LOCK_PREFIX}${groupId}`; - - try { - return await this.redlock.using([lockKey], LOCK_TTL_MS, async (lockSignal: AbortSignal) => { - const signal = AbortSignal.any([ - this.abortController.signal, - lockSignal, - ]); - - return await this.runJob(job, signal); - }); - } catch (error) { - if (error instanceof ExecutionError) { - // Lock could not be acquired - another job for this group is running - // Delay this job and let BullMQ retry later - // DelayedError tells BullMQ to delay without counting as a failed attempt - logger.debug(`Group ${groupId} locked, delaying job ${job.id}`); - await job.moveToDelayed(Date.now() + LOCK_RETRY_DELAY_MS, job.token); - throw new DelayedError(`Group ${groupId} locked, delaying job`); - } - throw error; - } - } - - private async runJob(job: Job, signal: AbortSignal) { - const id = job.data.jobId; - const logger = createJobLogger(id); - logger.debug(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId})`); - - const currentStatus = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { - id, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if ( - currentStatus.status !== RepoIndexingJobStatus.PENDING && - currentStatus.status !== RepoIndexingJobStatus.IN_PROGRESS - ) { - throw new Error(`Job ${id} is not in a valid state. Expected: ${RepoIndexingJobStatus.PENDING} or ${RepoIndexingJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - const { repo, type: jobType } = await this.db.repoIndexingJob.update({ - where: { - id, - }, - data: { - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - } - } - }, - select: { - type: true, - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobType); - this.promClient.pendingRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.activeRepoIndexJobs.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobType === RepoIndexingJobType.INDEX) { - const revisions = await this.indexRepository(repo, logger, signal); - - await this.db.repoIndexingJob.update({ - where: { id }, - data: { - metadata: { - indexedRevisions: revisions, - } satisfies RepoIndexingJobMetadata, - }, - }); - } else if (jobType === RepoIndexingJobType.CLEANUP) { - await this.cleanupRepository(repo, logger); - } - - } - - private async indexRepository(repo: RepoWithConnections, logger: Logger, signal: AbortSignal) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - - const metadata = repoMetadataSchema.parse(repo.metadata); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; - const authHeader = credentials?.authHeader ?? undefined; - - // If the repo path exists but it is not a valid git repository root, this indicates - // that the repository is in a bad state. To fix, we remove the directory and perform - // a fresh clone. - if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { - const isValidGitRepo = await isPathAValidGitRepoRoot({ - path: repoPath, - signal, - }); - - if (!isValidGitRepo && !isReadOnly) { - logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); - await rm(repoPath, { recursive: true, force: true }); - } - } - - if (existsSync(repoPath) && !isReadOnly) { - // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ - // write the clone URL (which could contain a auth token) to the - // `remote.origin.url` entry. For the upgrade scenario, we want - // to unset this key since it is no longer needed, hence this line. - // This will no-op if the key is already unset. - // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 - await unsetGitConfig({ - path: repoPath, - keys: ["remote.origin.url"], - signal, - }); - - logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); - const { durationMs } = await measure(() => fetchRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal, - })); - const fetchDuration_s = durationMs / 1000; - - logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); - - // Update the commit-graph after fetch. Force a full backfill the first time we - // see this repo after the --changed-paths rollout, so historical commits get - // Bloom filters. Subsequent fetches do a cheap incremental write. - const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; - if (needsBackfill) { - logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); - } - await writeCommitGraph({ - path: repoPath, - forceBackfill: needsBackfill, - signal, - }); - } else if (!isReadOnly) { - logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); - - const { durationMs } = await measure(() => cloneRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal - })); - const cloneDuration_s = durationMs / 1000; - - logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); - - // Write the commit-graph for the freshly cloned repo. - await writeCommitGraph({ - path: repoPath, - signal, - }); - } - - // Record that this repo's commit-graph now includes changed-path Bloom filters - // for its full history (either freshly written during clone, or backfilled above - // during fetch). - if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { - await this.db.repo.update({ - where: { id: repo.id }, - data: { - metadata: { - ...metadata, - commitGraphChangedPathsBackfilledAt: new Date().toISOString(), - } satisfies RepoMetadata, - }, - }); - } - - // Regardless of clone or fetch, always upsert the git config for the repo. - // This ensures that the git config is always up to date for whatever we - // have in the DB. - if (metadata.gitConfig && !isReadOnly) { - await upsertGitConfig({ - path: repoPath, - gitConfig: metadata.gitConfig, - signal, - }); - } - - const defaultBranch = await getLocalDefaultBranch({ - path: repoPath, - }); - - // Ensure defaultBranch has refs/heads/ prefix for consistent searching - const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') - ? `refs/heads/${defaultBranch}` - : defaultBranch; - - let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; - - if (metadata.branches) { - const branchGlobs = metadata.branches - const allBranches = await getBranches(repoPath); - const matchingBranches = - allBranches - .filter((branch) => micromatch.isMatch(branch, branchGlobs)) - .map((branch) => `refs/heads/${branch}`); - - revisions = [ - ...revisions, - ...matchingBranches - ]; - } - - if (metadata.tags) { - const tagGlobs = metadata.tags; - const allTags = await getTags(repoPath); - const matchingTags = - allTags - .filter((tag) => micromatch.isMatch(tag, tagGlobs)) - .map((tag) => `refs/tags/${tag}`); - - revisions = [ - ...revisions, - ...matchingTags - ]; - } - - // De-duplicate revisions to ensure we don't have duplicate branches/tags - revisions = [...new Set(revisions)]; - - // zoekt has a limit of 64 branches/tags to index. - if (revisions.length > 64) { - logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); - captureEvent('backend_revisions_truncated', { - repoId: repo.id, - revisionCount: revisions.length, - }); - revisions = revisions.slice(0, 64); - } - - logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); - try { - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal)); - const indexDuration_s = durationMs / 1000; - logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); - } catch (error) { - // Clean up any temporary shard files left behind by the failed indexing operation. - // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. - logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); - await cleanupTempShards(repo); - throw error; - } - - return revisions; - } - - private async cleanupRepository(repo: Repo, logger: Logger) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - if (existsSync(repoPath) && !isReadOnly) { - logger.debug(`Deleting repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); - for (const file of files) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting shard file ${filePath}`); - await rm(filePath, { force: true }); - } - } - - private async onJobCompleted(job: Job) { - try { - const logger = createJobLogger(job.data.jobId); - const jobData = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { id: job.data.jobId }, - include: { - repo: true, - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobData.type); - // @note: capture this before the update below, since the update sets indexedAt. - const isFirstIndex = jobData.repo.indexedAt === null; - - if (jobData.type === RepoIndexingJobType.INDEX) { - const { path: repoPath } = getRepoPath(jobData.repo); - const isEmpty = await isRepoEmpty({ path: repoPath }); - const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ - path: repoPath, - refName: 'HEAD', - }); - - const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); - const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); - - const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata); - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - indexedAt: new Date(), - indexedCommitHash: commitHash, - pushedAt: pushedAt, - metadata: { - ...(jobData.repo.metadata as RepoMetadata), - indexedRevisions: jobMetadata.indexedRevisions, - } satisfies RepoMetadata, - // @note: always update the default branch. While this field can be set - // during connection syncing, by setting it here we ensure that a) the - // default branch is as up to date as possible (since repo indexing happens - // more frequently than connection syncing) and b) for hosts where it is - // impossible to determine the default branch from the host's API - // (e.g., generic git url), we still set the default branch here. - defaultBranch: defaultBranch, - } - } - }, - include: { - repo: true, - } - }); - - logger.debug(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - else if (jobData.type === RepoIndexingJobType.CLEANUP) { - await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - } - }); - - const repo = await this.db.repo.delete({ - where: { id: jobData.repoId }, - }); - - logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - - // Track metrics for successful job - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobSuccessTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobData.type === RepoIndexingJobType.INDEX && isFirstIndex) { - captureEvent('backend_repo_first_indexed', { - repoId: job.data.repoId, - type: jobData.repo.external_codeHostType, - }); - } - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - - const jobLogger = createJobLogger(job.data.jobId); - const jobTypeLabel = getJobTypePrometheusLabel(job.data.type); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for repo ${job.data.repoName} (id: ${job.data.repoId}) failed. Retrying... Reason: ${error.message}`); - return; - } - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - } - } - }, - select: { repo: true } - }); - - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobFailTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - jobLogger.error(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Reason: ${error.message}`); - - captureEvent('backend_repo_index_job_failed', { - repoId: job.data.repoId, - jobType: job.data.type, - type: repo.external_codeHostType, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - // Scans the repos and index directories on disk and removes any entries - // that have no corresponding Repo record in the database. This handles - // edge cases where the DB and disk resources are out of sync. - private async cleanupOrphanedDiskResources() { - // --- Repo directories --- - // Dirs are named by repoId: DATA_CACHE_DIR/repos// - if (existsSync(REPOS_CACHE_DIR)) { - const entries = await readdir(REPOS_CACHE_DIR); - const repoIdToPath = new Map(); - for (const entry of entries) { - const repoPath = `${REPOS_CACHE_DIR}/${entry}`; - const repoId = getRepoIdFromPath(repoPath); - if (repoId !== undefined) { - repoIdToPath.set(repoId, repoPath); - } - } - - if (repoIdToPath.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToPath.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, repoPath] of repoIdToPath) { - if (!existingIds.has(repoId)) { - logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - } - } - } - - // --- Index shards --- - // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt - if (existsSync(INDEX_CACHE_DIR)) { - const entries = await readdir(INDEX_CACHE_DIR); - const repoIdToShards = new Map(); - for (const entry of entries) { - const repoId = getRepoIdFromShardFileName(entry); - if (repoId !== undefined) { - const shards = repoIdToShards.get(repoId) ?? []; - shards.push(entry); - repoIdToShards.set(repoId, shards); - } - } - - if (repoIdToShards.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToShards.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, shards] of repoIdToShards) { - if (!existingIds.has(repoId)) { - for (const entry of shards) { - const shardPath = `${INDEX_CACHE_DIR}/${entry}`; - logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); - await rm(shardPath, { force: true }); - } - } - } - } - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - // Locks will auto-expire via TTL, no need to manually release them - await this.queue.close(); - } -} - -const getJobTypePrometheusLabel = (type: RepoIndexingJobType) => type === RepoIndexingJobType.INDEX ? 'index' : 'cleanup'; diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts new file mode 100644 index 000000000..a2a66e77d --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.ts @@ -0,0 +1,351 @@ +import { PrismaClient, Repo } from "@sourcebot/db"; +import { createLogger, getRepoPath, JobLogSink, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { existsSync } from 'fs'; +import { readdir, rm } from 'fs/promises'; +import micromatch from 'micromatch'; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; +import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; +import { captureEvent } from './posthog.js'; +import { RepoWithConnections, Settings, Workload } from "./types.js"; +import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure } from './utils.js'; +import { cleanupTempShards, indexGitRepository } from './zoekt.js'; + +const LOG_TAG = 'repo-index-workload'; +const logger = createLogger(LOG_TAG); + +interface RepoIndexWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createRepoIndexWorkload = ({ + db, + settings, +}: RepoIndexWorkloadDependencies): Workload<'repo-index'> => ({ + queueSpec: REPO_INDEX_QUEUE, + concurrency: settings.maxRepoIndexingJobConcurrency, + process: async ({ data, logger: jobLogger, signal }) => { + const repo = await db.repo.findUniqueOrThrow({ + where: { id: data.repoId }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + jobLogger.debug(`Running ${data.type} job for repo ${repo.name} (id: ${repo.id})`); + + if (data.type === 'CLEANUP') { + await cleanupRepository(repo, jobLogger); + await db.repo.delete({ + where: { id: repo.id }, + }); + } else { + const isFirstIndex = repo.indexedAt === null; + const revisions = await indexRepository(db, settings, repo, jobLogger, signal); + const { path: repoPath } = getRepoPath(repo); + const isEmpty = await isRepoEmpty({ path: repoPath }); + const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ + path: repoPath, + refName: 'HEAD', + }); + const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); + const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); + const currentRepo = await db.repo.findUniqueOrThrow({ + where: { id: repo.id }, + select: { metadata: true }, + }); + + await db.repo.update({ + where: { id: repo.id }, + data: { + indexedAt: new Date(), + indexedCommitHash: commitHash, + pushedAt, + metadata: { + ...(currentRepo.metadata as RepoMetadata), + indexedRevisions: revisions, + } satisfies RepoMetadata, + defaultBranch, + }, + }); + + if (isFirstIndex) { + captureEvent('backend_repo_first_indexed', { + repoId: repo.id, + type: repo.external_codeHostType, + }); + } + } + }, +}); + +const indexRepository = async ( + db: PrismaClient, + settings: Settings, + repo: RepoWithConnections, + logger: JobLogSink, + signal: AbortSignal, +) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + + const metadata = repoMetadataSchema.parse(repo.metadata); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; + const authHeader = credentials?.authHeader ?? undefined; + + // If the repo path exists but it is not a valid git repository root, this indicates + // that the repository is in a bad state. To fix, we remove the directory and perform + // a fresh clone. + if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { + const isValidGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + signal, + }); + + if (!isValidGitRepo && !isReadOnly) { + logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); + await rm(repoPath, { recursive: true, force: true }); + } + } + + if (existsSync(repoPath) && !isReadOnly) { + // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ + // write the clone URL (which could contain a auth token) to the + // `remote.origin.url` entry. For the upgrade scenario, we want + // to unset this key since it is no longer needed, hence this line. + // This will no-op if the key is already unset. + // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 + await unsetGitConfig({ + path: repoPath, + keys: ["remote.origin.url"], + signal, + }); + + logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); + const { durationMs } = await measure(() => fetchRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal, + })); + const fetchDuration_s = durationMs / 1000; + + logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); + + // Update the commit-graph after fetch. Force a full backfill the first time we + // see this repo after the --changed-paths rollout, so historical commits get + // Bloom filters. Subsequent fetches do a cheap incremental write. + const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; + if (needsBackfill) { + logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); + } + await writeCommitGraph({ + path: repoPath, + forceBackfill: needsBackfill, + signal, + }); + } else if (!isReadOnly) { + logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); + + const { durationMs } = await measure(() => cloneRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal + })); + const cloneDuration_s = durationMs / 1000; + + logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); + + // Write the commit-graph for the freshly cloned repo. + await writeCommitGraph({ + path: repoPath, + signal, + }); + } + + // Record that this repo's commit-graph now includes changed-path Bloom filters + // for its full history (either freshly written during clone, or backfilled above + // during fetch). + if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { + await db.repo.update({ + where: { id: repo.id }, + data: { + metadata: { + ...metadata, + commitGraphChangedPathsBackfilledAt: new Date().toISOString(), + } satisfies RepoMetadata, + }, + }); + } + + // Regardless of clone or fetch, always upsert the git config for the repo. + // This ensures that the git config is always up to date for whatever we + // have in the DB. + if (metadata.gitConfig && !isReadOnly) { + await upsertGitConfig({ + path: repoPath, + gitConfig: metadata.gitConfig, + signal, + }); + } + + const defaultBranch = await getLocalDefaultBranch({ + path: repoPath, + }); + + // Ensure defaultBranch has refs/heads/ prefix for consistent searching + const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') + ? `refs/heads/${defaultBranch}` + : defaultBranch; + + let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; + + if (metadata.branches) { + const branchGlobs = metadata.branches + const allBranches = await getBranches(repoPath); + const matchingBranches = + allBranches + .filter((branch) => micromatch.isMatch(branch, branchGlobs)) + .map((branch) => `refs/heads/${branch}`); + + revisions = [ + ...revisions, + ...matchingBranches + ]; + } + + if (metadata.tags) { + const tagGlobs = metadata.tags; + const allTags = await getTags(repoPath); + const matchingTags = + allTags + .filter((tag) => micromatch.isMatch(tag, tagGlobs)) + .map((tag) => `refs/tags/${tag}`); + + revisions = [ + ...revisions, + ...matchingTags + ]; + } + + // De-duplicate revisions to ensure we don't have duplicate branches/tags + revisions = [...new Set(revisions)]; + + // zoekt has a limit of 64 branches/tags to index. + if (revisions.length > 64) { + logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); + captureEvent('backend_revisions_truncated', { + repoId: repo.id, + revisionCount: revisions.length, + }); + revisions = revisions.slice(0, 64); + } + + logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); + try { + const { durationMs } = await measure(() => indexGitRepository(repo, settings, revisions, signal)); + const indexDuration_s = durationMs / 1000; + logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); + } catch (error) { + // Clean up any temporary shard files left behind by the failed indexing operation. + // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. + logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); + await cleanupTempShards(repo); + throw error; + } + + return revisions; +}; + +const cleanupRepository = async (repo: Repo, logger: JobLogSink) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + if (existsSync(repoPath) && !isReadOnly) { + logger.debug(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const shardPrefix = getShardPrefix(repo.orgId, repo.id); + const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); + for (const file of files) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +// Scans the repos and index directories on disk and removes any entries +// that have no corresponding Repo record in the database. This handles +// edge cases where the DB and disk resources are out of sync. +export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { + // --- Repo directories --- + // Dirs are named by repoId: DATA_CACHE_DIR/repos// + if (existsSync(REPOS_CACHE_DIR)) { + const entries = await readdir(REPOS_CACHE_DIR); + const repoIdToPath = new Map(); + for (const entry of entries) { + const repoPath = `${REPOS_CACHE_DIR}/${entry}`; + const repoId = getRepoIdFromPath(repoPath); + if (repoId !== undefined) { + repoIdToPath.set(repoId, repoPath); + } + } + + if (repoIdToPath.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToPath.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, repoPath] of repoIdToPath) { + if (!existingIds.has(repoId)) { + logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + } + } + } + + // --- Index shards --- + // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt + if (existsSync(INDEX_CACHE_DIR)) { + const entries = await readdir(INDEX_CACHE_DIR); + const repoIdToShards = new Map(); + for (const entry of entries) { + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + const shards = repoIdToShards.get(repoId) ?? []; + shards.push(entry); + repoIdToShards.set(repoId, shards); + } + } + + if (repoIdToShards.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToShards.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, shards] of repoIdToShards) { + if (!existingIds.has(repoId)) { + for (const entry of shards) { + const shardPath = `${INDEX_CACHE_DIR}/${entry}`; + logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); + await rm(shardPath, { force: true }); + } + } + } + } + } +}; diff --git a/packages/backend/src/types/redlock.d.ts b/packages/backend/src/types/redlock.d.ts deleted file mode 100644 index 63c6b5a9b..000000000 --- a/packages/backend/src/types/redlock.d.ts +++ /dev/null @@ -1,95 +0,0 @@ -// Type declarations for redlock -// The redlock package's exports field doesn't include types, so TypeScript can't resolve them. -// This file re-exports the types from the actual .d.ts file. - -declare module 'redlock' { - import { EventEmitter } from "events"; - import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis"; - - type Client = IORedisClient | IORedisCluster; - - export type ClientExecutionResult = { - client: Client; - vote: "for"; - value: number; - } | { - client: Client; - vote: "against"; - error: Error; - }; - - export type ExecutionStats = { - readonly membershipSize: number; - readonly quorumSize: number; - readonly votesFor: Set; - readonly votesAgainst: Map; - }; - - export type ExecutionResult = { - attempts: ReadonlyArray>; - }; - - export interface Settings { - readonly driftFactor: number; - readonly retryCount: number; - readonly retryDelay: number; - readonly retryJitter: number; - readonly automaticExtensionThreshold: number; - } - - export class ResourceLockedError extends Error { - readonly message: string; - constructor(message: string); - } - - export class ExecutionError extends Error { - readonly message: string; - readonly attempts: ReadonlyArray>; - constructor(message: string, attempts: ReadonlyArray>); - } - - export class Lock { - readonly redlock: Redlock; - readonly resources: string[]; - readonly value: string; - readonly attempts: ReadonlyArray>; - expiration: number; - constructor(redlock: Redlock, resources: string[], value: string, attempts: ReadonlyArray>, expiration: number); - release(): Promise; - extend(duration: number): Promise; - } - - export type RedlockAbortSignal = AbortSignal & { - error?: Error; - }; - - export default class Redlock extends EventEmitter { - readonly clients: Set; - readonly settings: Settings; - readonly scripts: { - readonly acquireScript: { - value: string; - hash: string; - }; - readonly extendScript: { - value: string; - hash: string; - }; - readonly releaseScript: { - value: string; - hash: string; - }; - }; - constructor(clients: Iterable, settings?: Partial, scripts?: { - readonly acquireScript?: string | ((script: string) => string); - readonly extendScript?: string | ((script: string) => string); - readonly releaseScript?: string | ((script: string) => string); - }); - quit(): Promise; - acquire(resources: string[], duration: number, settings?: Partial): Promise; - release(lock: Lock, settings?: Partial): Promise; - extend(existing: Lock, duration: number, settings?: Partial): Promise; - using(resources: string[], duration: number, settings: Partial, routine?: (signal: RedlockAbortSignal) => Promise): Promise; - using(resources: string[], duration: number, routine: (signal: RedlockAbortSignal) => Promise): Promise; - } -} diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index e50ee0ed5..45c91226b 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,7 +1,7 @@ import { Logger } from "winston"; import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; -import { getTokenFromConfig } from "@sourcebot/shared"; +import { getTokenFromConfig, JobLogSink } from "@sourcebot/shared"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GithubAppManager } from "./ee/githubAppManager.js"; @@ -114,7 +114,7 @@ export const fetchWithRetry = async ( // fetch the token here using the connections from the repo. Multiple connections could be referencing this repo, and each // may have their own token. This method will just pick the first connection that has a token (if one exists) and uses that. This // may technically cause syncing to fail if that connection's token just so happens to not have access to the repo it's referencing. -export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: Logger): Promise => { +export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: JobLogSink): Promise => { // If we have github apps configured we assume that we must use them for github service auth if (repo.external_codeHostType === 'github' && await hasEntitlement('github-app') && GithubAppManager.getInstance().appsConfigured()) { logger?.debug(`Using GitHub App for service auth for repo ${repo.displayName} hosted at ${repo.external_codeHostUrl}`); diff --git a/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql b/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql new file mode 100644 index 000000000..ab019a779 --- /dev/null +++ b/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql @@ -0,0 +1,22 @@ +/* + Warnings: + + - You are about to drop the column `latestIndexingJobStatus` on the `Repo` table. All the data in the column will be lost. + - You are about to drop the `RepoIndexingJob` table. If the table is not empty, all the data it contains will be lost. + +*/ +-- DropForeignKey +ALTER TABLE "RepoIndexingJob" DROP CONSTRAINT "RepoIndexingJob_repoId_fkey"; + +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "latestIndexingJobStatus", +ADD COLUMN "latestIndexingJobId" TEXT; + +-- DropTable +DROP TABLE "RepoIndexingJob"; + +-- DropEnum +DROP TYPE "RepoIndexingJobStatus"; + +-- DropEnum +DROP TYPE "RepoIndexingJobType"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 336a306e9..535833061 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -65,10 +65,10 @@ model Repo { permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. - jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. + latestIndexingJobId String? + indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). - latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. pushedAt DateTime? /// The timestamp of the most recent commit across all branches. external_id String /// The id of the repo in the external service @@ -86,35 +86,6 @@ model Repo { @@index([indexedAt]) } -enum RepoIndexingJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -enum RepoIndexingJobType { - INDEX - CLEANUP -} - -model RepoIndexingJob { - id String @id @default(cuid()) - type RepoIndexingJobType - status RepoIndexingJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts - - errorMessage String? - - repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) - repoId Int - - @@index([repoId, type, status]) -} - enum RepoPermissionSyncJobStatus { PENDING IN_PROGRESS diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 609bcdcc7..5540f97b4 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -2,7 +2,6 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; const NUM_REPOS = 1000; -const NUM_INDEXING_JOBS_PER_REPO = 10000; const NUM_PERMISSION_JOBS_PER_REPO = 10000; export const injectRepoData: Script = { @@ -37,8 +36,6 @@ export const injectRepoData: Script = { console.log(`Creating ${NUM_REPOS} repos...`); const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const; - const indexingJobTypes = ['INDEX', 'CLEANUP'] as const; - for (let i = 0; i < NUM_REPOS; i++) { const repo = await prisma.repo.create({ data: { @@ -71,23 +68,8 @@ export const injectRepoData: Script = { } }); } - - for (let j = 0; j < NUM_INDEXING_JOBS_PER_REPO; j++) { - const status = statuses[Math.floor(Math.random() * statuses.length)]; - const type = indexingJobTypes[Math.floor(Math.random() * indexingJobTypes.length)]; - await prisma.repoIndexingJob.create({ - data: { - repoId: repo.id, - type, - status, - completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, - errorMessage: status === 'FAILED' ? 'Mock indexing error' : null, - metadata: {} - } - }); - } } - console.log(`Created ${NUM_REPOS} repos with associated jobs.`); + console.log(`Created ${NUM_REPOS} repos with associated permission jobs.`); } -}; \ No newline at end of file +}; diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index ee165d5b8..c73b1fb5d 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -2,6 +2,7 @@ import type { Redis } from 'ioredis'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import type { QueueSpec } from './queue.js'; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; +import { REPO_INDEX_QUEUE } from './queue.js'; const queueMocks = vi.hoisted(() => ({ add: vi.fn(), @@ -96,6 +97,39 @@ describe('BullMQClient', () => { expect(onEnqueued).not.toHaveBeenCalled(); }); + test('stores the latest repo indexing job id when enqueueing a repo job', async () => { + const repoUpdate = vi.fn(); + const client = new BullMQClient(redis, { + repo: { + update: repoUpdate, + }, + } as unknown as PrismaClient); + + const result = await client.enqueue(REPO_INDEX_QUEUE, { + repoId: 42, + type: 'INDEX', + }); + + expect(queueMocks.add).toHaveBeenCalledWith( + 'repo-index', + { + repoId: 42, + type: 'INDEX', + }, + expect.objectContaining({ + deduplication: { id: 'repo:42' }, + }), + ); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobId: result, + }, + }); + }); + test.each([ ['waiting', 'PENDING'], ['waiting-children', 'PENDING'], diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 51d4c9077..f465e677b 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -19,13 +19,11 @@ export type { } from "./entitlements.js"; export type { RepoMetadata, - RepoIndexingJobMetadata, IdentityProviderType, LicenseStatus, } from "./types.js"; export { repoMetadataSchema, - repoIndexingJobMetadataSchema, } from "./types.js"; export { base64Decode, @@ -101,6 +99,7 @@ export type { export { CONNECTION_QUEUE, RECONCILIATION_QUEUE, + REPO_INDEX_QUEUE, } from "./queue.js"; export { BullMQClient, diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index fb375d711..585d1e457 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -12,6 +12,10 @@ interface QueueRegistry { orgId: number }, 'reconciliation': EmptyJobData, + 'repo-index': { + repoId: number, + type: 'INDEX' | 'CLEANUP', + }, } export const CONNECTION_QUEUE: QueueSpec<'connection'> = { @@ -50,6 +54,31 @@ export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { }, }; +export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { + name: 'repo-index', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + onEnqueued: async ({ + prisma, + data: { repoId }, + jobId, + }) => { + await prisma.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobId: jobId, + }, + }); + }, + dedupKey: (data) => `repo:${data.repoId}`, +}; + export interface QueueSpec { name: TName; dedupKey?(data: DataOf): string; diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 5951bab6b..9f93637c9 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -63,15 +63,6 @@ export const repoMetadataSchema = z.object({ export type RepoMetadata = z.infer; -export const repoIndexingJobMetadataSchema = z.object({ - /** - * A list of revisions that were indexed for the repo. - */ - indexedRevisions: z.array(z.string()).optional(), -}); - -export type RepoIndexingJobMetadata = z.infer; - export type IdentityProviderType = IdentityProviderConfig['provider']; // @see: https://docs.stripe.com/api/subscriptions/object#subscription_object-status diff --git a/packages/web/src/app/(app)/chat/chatLandingPage.tsx b/packages/web/src/app/(app)/chat/chatLandingPage.tsx index b78e28e10..fdebccf77 100644 --- a/packages/web/src/app/(app)/chat/chatLandingPage.tsx +++ b/packages/web/src/app/(app)/chat/chatLandingPage.tsx @@ -1,4 +1,4 @@ -import { getRepos, getReposStats, getSearchContexts } from "@/actions"; +import { getRepos, getSearchContexts } from "@/actions"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; @@ -35,8 +35,6 @@ export async function ChatLandingPage() { take: 10, }); - const repoStats = await getReposStats(); - if (isServiceError(allRepos)) { throw new ServiceErrorException(allRepos); } @@ -49,10 +47,6 @@ export async function ChatLandingPage() { throw new ServiceErrorException(carouselRepos); } - if (isServiceError(repoStats)) { - throw new ServiceErrorException(repoStats); - } - const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { try { return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; @@ -84,7 +78,8 @@ export async function ChatLandingPage() {
diff --git a/packages/web/src/app/(app)/repos/layout.tsx b/packages/web/src/app/(app)/repos/layout.tsx index 88738d22e..61f3cd34e 100644 --- a/packages/web/src/app/(app)/repos/layout.tsx +++ b/packages/web/src/app/(app)/repos/layout.tsx @@ -1,7 +1,3 @@ -import { getReposStats } from "@/actions"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; - interface LayoutProps { children: React.ReactNode; } @@ -11,11 +7,6 @@ export default async function Layout( ) { const { children } = props; - const repoStats = await getReposStats(); - if (isServiceError(repoStats)) { - throw new ServiceErrorException(repoStats); - } - return (
diff --git a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx index e9236607e..a6d072f29 100644 --- a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx +++ b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx @@ -5,7 +5,7 @@ import { SyntaxReferenceGuideHint } from "../../components/syntaxReferenceGuideH import Link from "next/link" import { SearchBar } from "../../components/searchBar" import { SearchModeSelector } from "../../components/searchModeSelector" -import { getRepos, getReposStats } from "@/actions" +import { getRepos } from "@/actions" import { ServiceErrorException } from "@/lib/serviceError" import { isServiceError } from "@/lib/utils" @@ -25,10 +25,7 @@ export const SearchLandingPage = async ({ take: 10, }); - const repoStats = await getReposStats(); - if (isServiceError(carouselRepos)) throw new ServiceErrorException(carouselRepos); - if (isServiceError(repoStats)) throw new ServiceErrorException(repoStats); return (
@@ -55,7 +52,8 @@ export const SearchLandingPage = async ({
diff --git a/yarn.lock b/yarn.lock index ece7d1c39..0982a01a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9194,7 +9194,6 @@ __metadata: p-limit: "npm:^7.2.0" posthog-node: "npm:^5.24.15" prom-client: "npm:^15.1.3" - redlock: "npm:5.0.0-beta.2" simple-git: "npm:^3.36.0" tsc-watch: "npm:^6.2.0" tsx: "npm:^4.21.0" @@ -18863,7 +18862,7 @@ __metadata: languageName: node linkType: hard -"node-abort-controller@npm:^3.0.1, node-abort-controller@npm:^3.1.1": +"node-abort-controller@npm:^3.1.1": version: 3.1.1 resolution: "node-abort-controller@npm:3.1.1" checksum: 10c0/f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 @@ -20887,15 +20886,6 @@ __metadata: languageName: node linkType: hard -"redlock@npm:5.0.0-beta.2": - version: 5.0.0-beta.2 - resolution: "redlock@npm:5.0.0-beta.2" - dependencies: - node-abort-controller: "npm:^3.0.1" - checksum: 10c0/6664a1b7807ec0ceb223d8f50bf087ac9a6df6356329c47b4374c3a7e4cbcb24a3045d45735dbc74422fc399657baf533e24c0d6528c141c9dd14701880d4fe4 - languageName: node - linkType: hard - "reflect.getprototypeof@npm:^1.0.6, reflect.getprototypeof@npm:^1.0.9": version: 1.0.10 resolution: "reflect.getprototypeof@npm:1.0.10" From 7336c5e1297cdd3ccc5b190f03e5aa7f45250bf7 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 15:53:55 -0700 Subject: [PATCH 06/10] further wip --- docs/snippets/schemas/v3/index.schema.mdx | 26 +- packages/backend/package.json | 7 +- packages/backend/src/api.ts | 15 +- packages/backend/src/index.ts | 3 +- packages/backend/src/jobManager.ts | 12 +- .../backend/src/reconciliationWorkload.ts | 2 +- packages/schemas/src/v3/index.schema.ts | 26 +- packages/schemas/src/v3/index.type.ts | 11 +- packages/shared/package.json | 4 +- packages/shared/src/constants.ts | 10 +- schemas/v3/index.json | 13 +- yarn.lock | 376 +++++++++++------- 12 files changed, 313 insertions(+), 192 deletions(-) diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 0a25f6917..4f38c3429 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -42,19 +42,20 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -96,12 +97,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, @@ -228,19 +229,20 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -282,12 +284,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, diff --git a/packages/backend/package.json b/packages/backend/package.json index 8d9f61aaf..7788e5093 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -22,6 +22,9 @@ "vitest": "^4.1.4" }, "dependencies": { + "@bull-board/api": "6.11.2", + "@bull-board/express": "6.11.2", + "@bull-board/ui": "6.11.2", "@coderabbitai/bitbucket": "^1.1.3", "@gitbeaker/rest": "^40.5.1", "@octokit/app": "^16.1.1", @@ -35,7 +38,7 @@ "@types/express": "^5.0.0", "argparse": "^2.0.1", "azure-devops-node-api": "^15.1.1", - "bullmq": "^5.34.10", + "bullmq": "^5.81.3", "chokidar": "^4.0.3", "cross-fetch": "^4.0.0", "dotenv": "^16.4.5", @@ -46,7 +49,7 @@ "gitea-js": "^1.22.0", "glob": "^11.1.0", "http-status-codes": "^2.3.0", - "ioredis": "^5.4.2", + "ioredis": "^5.11.1", "lowdb": "^7.0.1", "micromatch": "^4.0.8", "p-limit": "^7.2.0", diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index fd1d91ba5..a04d7bd9f 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -1,4 +1,8 @@ import { createLogger, env } from '@sourcebot/shared'; +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js'; +import { ExpressAdapter } from '@bull-board/express'; +import { Queue } from 'bullmq'; import express, { Request, Response } from 'express'; import 'express-async-errors'; import * as http from "http"; @@ -12,11 +16,19 @@ const PORT = Number(workerApiUrl.port) || (workerApiUrl.protocol === "https:" ? export class Api { private server: http.Server; - constructor(promClient: PromClient) { + constructor(promClient: PromClient, queues: Queue[]) { const app = express(); app.use(express.json()); app.use(express.urlencoded({ extended: true })); + const bullBoardAdapter = new ExpressAdapter(); + bullBoardAdapter.setBasePath('/admin/queues'); + createBullBoard({ + queues: queues.map(queue => new BullMQAdapter(queue, { readOnlyMode: true })), + serverAdapter: bullBoardAdapter, + }); + app.use('/admin/queues', bullBoardAdapter.getRouter()); + // Prometheus metrics endpoint app.use('/metrics', async (_req: Request, res: Response) => { res.set('Content-Type', promClient.registry.contentType); @@ -26,6 +38,7 @@ export class Api { this.server = app.listen(PORT, () => { logger.debug(`API server is running on port ${PORT}`); + logger.debug(`Bull Board is available at ${workerApiUrl.origin}/admin/queues`); }); } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 73db4fc3f..8a8527189 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -48,7 +48,6 @@ if (await hasEntitlement('github-app')) { } const promClient = new PromClient(); -const api = new Api(promClient); logger.info('Worker started.'); @@ -67,6 +66,8 @@ jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); jobManager.register(repoIndexWorkload); +const api = new Api(promClient, jobManager.getQueues()); + await cleanupOrphanedRepoResources(prisma); await jobManager.start(); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index cd49b5c76..b05ab4264 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,6 +1,6 @@ import * as Sentry from "@sentry/node"; import { BullMQClient, createBullMQJobLogger, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; -import { Job, Worker } from "bullmq"; +import { Job, Queue, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; @@ -63,6 +63,12 @@ export class BullMQJobManager implements JobManager { this.workloads.set(name, workload); } + getQueues(): Queue[] { + return [...this.workloads.values()].map((workload) => + this.bullmqClient.getQueue(workload.queueSpec), + ); + } + async start(): Promise { if (this.workloads.size === 0) { logger.debug('start() called with nothing registered; nothing to do'); @@ -117,7 +123,7 @@ export class BullMQJobManager implements JobManager { `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, ); const lifecycleContext = this.jobLifecycleContext(job); - jobLogger.info(`Started workload "${spec.name}"`); + jobLogger.debug(`Started workload "${spec.name}"`); try { await workload.onStarted?.(lifecycleContext); @@ -128,7 +134,7 @@ export class BullMQJobManager implements JobManager { updateProgress: (progress) => job.updateProgress(progress), trigger: (target, data) => this.trigger(target, data), }); - jobLogger.info(`Completed workload "${spec.name}"`); + jobLogger.debug(`Completed workload "${spec.name}"`); return result; } catch (error) { jobLogger.error(`Workload "${spec.name}" attempt failed`, error); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index e36e7c975..f34008b2a 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -12,7 +12,7 @@ export const createReconciliationWorkload = ({ settings, }: ReconciliationWorkloadDependencies): Workload<'reconciliation'> => ({ concurrency: 1, - schedule: { every: '15m' }, + schedule: { every: '10s' }, queueSpec: RECONCILIATION_QUEUE, process: async ({ logger, trigger }) => { const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index c55c72a05..228a7f9ce 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -41,19 +41,20 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -95,12 +96,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, @@ -227,19 +228,20 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -281,12 +283,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 3bede3def..833b5aa15 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -110,15 +110,16 @@ export interface Settings { reindexRepoPollingIntervalMs?: number; /** * @deprecated - * The number of connection sync jobs to run concurrently. Defaults to 8. + * The number of connection sync jobs to run concurrently. Defaults to 2. */ maxConnectionSyncJobConcurrency?: number; /** - * The number of repo indexing jobs to run concurrently. Defaults to 8. + * The number of repo indexing jobs to run concurrently. Defaults to 2. */ maxRepoIndexingJobConcurrency?: number; /** - * The number of repo GC jobs to run concurrently. Defaults to 8. + * @deprecated + * The number of repo GC jobs to run concurrently. Defaults to 2. */ maxRepoGarbageCollectionJobConcurrency?: number; /** @@ -153,11 +154,11 @@ export interface Settings { */ experiment_userDrivenPermissionSyncIntervalMs?: number; /** - * The number of account permission sync jobs to run concurrently. Defaults to 8. + * The number of account permission sync jobs to run concurrently. Defaults to 2. */ maxAccountPermissionSyncJobConcurrency?: number; /** - * The number of repo permission sync jobs to run concurrently. Defaults to 8. + * The number of repo permission sync jobs to run concurrently. Defaults to 2. */ maxRepoPermissionSyncJobConcurrency?: number; } diff --git a/packages/shared/package.json b/packages/shared/package.json index e865230c3..7967ad2a6 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -19,8 +19,8 @@ "@sourcebot/schemas": "workspace:*", "@t3-oss/env-core": "^0.13.10", "ajv": "^8.17.1", - "bullmq": "^5.34.10", - "ioredis": "^5.4.2", + "bullmq": "^5.81.3", + "ioredis": "^5.11.1", "micromatch": "^4.0.8", "strip-json-comments": "^5.0.1", "triple-beam": "^1.4.1", diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index c299ef1cc..fce116549 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -23,9 +23,9 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second reindexRepoPollingIntervalMs: 1000 * 1, // 1 second - maxConnectionSyncJobConcurrency: 8, - maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, + maxConnectionSyncJobConcurrency: 2, + maxRepoIndexingJobConcurrency: 2, + maxRepoGarbageCollectionJobConcurrency: 2, repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead @@ -33,8 +33,8 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) - maxAccountPermissionSyncJobConcurrency: 8, - maxRepoPermissionSyncJobConcurrency: 8, + maxAccountPermissionSyncJobConcurrency: 2, + maxRepoPermissionSyncJobConcurrency: 2, } export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: CodeHostType[] = [ diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 6dc6255f1..887d40eb7 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -40,19 +40,20 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", "minimum": 1, "deprecated": true }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -94,12 +95,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", "minimum": 1 } }, diff --git a/yarn.lock b/yarn.lock index 0982a01a2..99721e6bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1340,6 +1340,38 @@ __metadata: languageName: node linkType: hard +"@bull-board/api@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/api@npm:6.11.2" + dependencies: + redis-info: "npm:^3.1.0" + peerDependencies: + "@bull-board/ui": 6.11.2 + checksum: 10c0/d6a82bdd598d41c4e09dd4e8a001f49e87ed40aa535ca841c5430590f0df206ae49ac8835c2832b00cd6bdb413ef89ea7083a5ffc1b779a307744ea69d83c76d + languageName: node + linkType: hard + +"@bull-board/express@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/express@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" + ejs: "npm:^3.1.10" + express: "npm:^4.21.1 || ^5.0.0" + checksum: 10c0/36cf6fb63f51f095934ba6167be8704f45d881c882d56e11e34f9b60e0dac16bc989553292942797af0f62beac2c10f116a852988dadb1bc07cd06d48e1b8ae3 + languageName: node + linkType: hard + +"@bull-board/ui@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/ui@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + checksum: 10c0/7ddcb49222c09f32f63d9a55f000445cd59dd6c73620f5e30ffcf04cfa8727c91918485a7c3d17c709d2b721b4a8d32d2026c2a7f208953cd3facd602e595f03 + languageName: node + linkType: hard + "@cfworker/json-schema@npm:^4.0.2": version: 4.1.1 resolution: "@cfworker/json-schema@npm:4.1.1" @@ -3220,10 +3252,10 @@ __metadata: languageName: node linkType: hard -"@ioredis/commands@npm:^1.1.1": - version: 1.2.0 - resolution: "@ioredis/commands@npm:1.2.0" - checksum: 10c0/a5d3c29dd84d8a28b7c67a441ac1715cbd7337a7b88649c0f17c345d89aa218578d2b360760017c48149ef8a70f44b051af9ac0921a0622c2b479614c4f65b36 +"@ioredis/commands@npm:1.10.0": + version: 1.10.0 + resolution: "@ioredis/commands@npm:1.10.0" + checksum: 10c0/baf91e62d0e64ef2b5f7ca4413dc2456fe250e87483beac4a1c8ef1fe5ad0d2fcdeb9b89d4556d8ef6c7455c64a964359d729601fdb06b2f4c76c35dd59afa99 languageName: node linkType: hard @@ -3805,44 +3837,44 @@ __metadata: languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -9159,6 +9191,9 @@ __metadata: version: 0.0.0-use.local resolution: "@sourcebot/backend@workspace:packages/backend" dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/express": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" "@coderabbitai/bitbucket": "npm:^1.1.3" "@gitbeaker/rest": "npm:^40.5.1" "@octokit/app": "npm:^16.1.1" @@ -9175,7 +9210,7 @@ __metadata: "@types/node": "npm:^22.7.5" argparse: "npm:^2.0.1" azure-devops-node-api: "npm:^15.1.1" - bullmq: "npm:^5.34.10" + bullmq: "npm:^5.81.3" chokidar: "npm:^4.0.3" cross-env: "npm:^7.0.3" cross-fetch: "npm:^4.0.0" @@ -9187,7 +9222,7 @@ __metadata: gitea-js: "npm:^1.22.0" glob: "npm:^11.1.0" http-status-codes: "npm:^2.3.0" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" json-schema-to-typescript: "npm:^15.0.4" lowdb: "npm:^7.0.1" micromatch: "npm:^4.0.8" @@ -9272,9 +9307,9 @@ __metadata: "@types/micromatch": "npm:^4.0.9" "@types/node": "npm:^22.7.5" ajv: "npm:^8.17.1" - bullmq: "npm:^5.34.10" + bullmq: "npm:^5.81.3" cross-env: "npm:^7.0.3" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" micromatch: "npm:^4.0.8" strip-json-comments: "npm:^5.0.1" triple-beam: "npm:^1.4.1" @@ -11592,7 +11627,7 @@ __metadata: languageName: node linkType: hard -"async@npm:^3.2.3": +"async@npm:^3.2.3, async@npm:^3.2.6": version: 3.2.6 resolution: "async@npm:3.2.6" checksum: 10c0/36484bb15ceddf07078688d95e27076379cc2f87b10c03b6dd8a83e89475a3c8df5848859dd06a4c95af1e4c16fc973de0171a77f18ea00be899aca2a4f85e70 @@ -11848,6 +11883,15 @@ __metadata: languageName: node linkType: hard +"brace-expansion@npm:^2.0.1": + version: 2.1.4 + resolution: "brace-expansion@npm:2.1.4" + dependencies: + balanced-match: "npm:^1.0.0" + checksum: 10c0/6c0a0e2573eac1dc565b52b1e1bfbeba39bf1830d106ebbc61ff1eaefcf610e9111cd3baa091addd18e33292135c99e019d3184d966389d85dc958fdcdc1449f + languageName: node + linkType: hard + "brace-expansion@npm:^2.0.3": version: 2.1.0 resolution: "brace-expansion@npm:2.1.0" @@ -11896,18 +11940,22 @@ __metadata: languageName: node linkType: hard -"bullmq@npm:^5.34.10": - version: 5.44.3 - resolution: "bullmq@npm:5.44.3" +"bullmq@npm:^5.81.3": + version: 5.81.3 + resolution: "bullmq@npm:5.81.3" dependencies: - cron-parser: "npm:^4.9.0" - ioredis: "npm:^5.4.1" - msgpackr: "npm:^1.11.2" - node-abort-controller: "npm:^3.1.1" - semver: "npm:^7.5.4" - tslib: "npm:^2.0.0" - uuid: "npm:^9.0.0" - checksum: 10c0/2785929ef59645980e3981d6f5e3f7ef25ef0e8488281a7aa48e89c25a89ff5dde0431051f48b2edadd9675c5b780c02577724c33d5d31762009820b8d404cdb + cron-parser: "npm:4.9.0" + ioredis: "npm:5.11.1" + msgpackr: "npm:2.0.5" + node-abort-controller: "npm:3.1.1" + semver: "npm:7.8.5" + tslib: "npm:2.8.1" + peerDependencies: + redis: ">=5.0.0" + peerDependenciesMeta: + redis: + optional: true + checksum: 10c0/e28abf1f37191966d0204e8717040f99807749579ba34e6cd8ac4698f3daa0327be094e52fa52c6aab991f4cd4cdd338b6c950134ec083be166c19f82d96cde1 languageName: node linkType: hard @@ -12211,10 +12259,10 @@ __metadata: languageName: node linkType: hard -"cluster-key-slot@npm:^1.1.0": - version: 1.1.2 - resolution: "cluster-key-slot@npm:1.1.2" - checksum: 10c0/d7d39ca28a8786e9e801eeb8c770e3c3236a566625d7299a47bb71113fb2298ce1039596acb82590e598c52dbc9b1f088c8f587803e697cb58e1867a95ff94d3 +"cluster-key-slot@npm:1.1.1": + version: 1.1.1 + resolution: "cluster-key-slot@npm:1.1.1" + checksum: 10c0/079b1ae86b20e2d53308a877b08de5e830722a45c07810569d0dab4955bed569da33ac9f79998289d014adf02cca7223a0647cb0ee6548a12ab3c4f9beac1377 languageName: node linkType: hard @@ -12752,7 +12800,7 @@ __metadata: languageName: node linkType: hard -"cron-parser@npm:^4.9.0": +"cron-parser@npm:4.9.0": version: 4.9.0 resolution: "cron-parser@npm:4.9.0" dependencies: @@ -13370,6 +13418,18 @@ __metadata: languageName: node linkType: hard +"debug@npm:4.4.3, debug@npm:^4.4.3, debug@npm:~4.4.1": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + "debug@npm:^3.2.7": version: 3.2.7 resolution: "debug@npm:3.2.7" @@ -13391,18 +13451,6 @@ __metadata: languageName: node linkType: hard -"debug@npm:^4.4.3, debug@npm:~4.4.1": - version: 4.4.3 - resolution: "debug@npm:4.4.3" - dependencies: - ms: "npm:^2.1.3" - peerDependenciesMeta: - supports-color: - optional: true - checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 - languageName: node - linkType: hard - "debug@npm:~4.3.2": version: 4.3.7 resolution: "debug@npm:4.3.7" @@ -13509,7 +13557,7 @@ __metadata: languageName: node linkType: hard -"denque@npm:^2.1.0": +"denque@npm:2.1.0": version: 2.1.0 resolution: "denque@npm:2.1.0" checksum: 10c0/f9ef81aa0af9c6c614a727cb3bd13c5d7db2af1abf9e6352045b86e85873e629690f6222f4edd49d10e4ccf8f078bbeec0794fafaf61b659c0589d0c511ec363 @@ -13782,6 +13830,17 @@ __metadata: languageName: node linkType: hard +"ejs@npm:^3.1.10": + version: 3.1.10 + resolution: "ejs@npm:3.1.10" + dependencies: + jake: "npm:^10.8.5" + bin: + ejs: bin/cli.js + checksum: 10c0/52eade9e68416ed04f7f92c492183340582a36482836b11eab97b159fcdcfdedc62233a1bf0bf5e5e1851c501f2dca0e2e9afd111db2599e4e7f53ee29429ae1 + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.5.73": version: 1.5.123 resolution: "electron-to-chromium@npm:1.5.123" @@ -14795,6 +14854,42 @@ __metadata: languageName: node linkType: hard +"express@npm:^4.21.1 || ^5.0.0, express@npm:^5.2.1": + version: 5.2.1 + resolution: "express@npm:5.2.1" + dependencies: + accepts: "npm:^2.0.0" + body-parser: "npm:^2.2.1" + content-disposition: "npm:^1.0.0" + content-type: "npm:^1.0.5" + cookie: "npm:^0.7.1" + cookie-signature: "npm:^1.2.1" + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + finalhandler: "npm:^2.1.0" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.0" + merge-descriptors: "npm:^2.0.0" + mime-types: "npm:^3.0.0" + on-finished: "npm:^2.4.1" + once: "npm:^1.4.0" + parseurl: "npm:^1.3.3" + proxy-addr: "npm:^2.0.7" + qs: "npm:^6.14.0" + range-parser: "npm:^1.2.1" + router: "npm:^2.2.0" + send: "npm:^1.1.0" + serve-static: "npm:^2.2.0" + statuses: "npm:^2.0.1" + type-is: "npm:^2.0.1" + vary: "npm:^1.1.2" + checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 + languageName: node + linkType: hard + "express@npm:^4.21.2": version: 4.21.2 resolution: "express@npm:4.21.2" @@ -14834,42 +14929,6 @@ __metadata: languageName: node linkType: hard -"express@npm:^5.2.1": - version: 5.2.1 - resolution: "express@npm:5.2.1" - dependencies: - accepts: "npm:^2.0.0" - body-parser: "npm:^2.2.1" - content-disposition: "npm:^1.0.0" - content-type: "npm:^1.0.5" - cookie: "npm:^0.7.1" - cookie-signature: "npm:^1.2.1" - debug: "npm:^4.4.0" - depd: "npm:^2.0.0" - encodeurl: "npm:^2.0.0" - escape-html: "npm:^1.0.3" - etag: "npm:^1.8.1" - finalhandler: "npm:^2.1.0" - fresh: "npm:^2.0.0" - http-errors: "npm:^2.0.0" - merge-descriptors: "npm:^2.0.0" - mime-types: "npm:^3.0.0" - on-finished: "npm:^2.4.1" - once: "npm:^1.4.0" - parseurl: "npm:^1.3.3" - proxy-addr: "npm:^2.0.7" - qs: "npm:^6.14.0" - range-parser: "npm:^1.2.1" - router: "npm:^2.2.0" - send: "npm:^1.1.0" - serve-static: "npm:^2.2.0" - statuses: "npm:^2.0.1" - type-is: "npm:^2.0.1" - vary: "npm:^1.1.2" - checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 - languageName: node - linkType: hard - "extend@npm:^3.0.0, extend@npm:^3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" @@ -15090,6 +15149,15 @@ __metadata: languageName: node linkType: hard +"filelist@npm:^1.0.4": + version: 1.0.6 + resolution: "filelist@npm:1.0.6" + dependencies: + minimatch: "npm:^5.0.1" + checksum: 10c0/6ee725bec3e1936d680a45f14439b224d9f7c71658c145addcf551dd82f03d608522eb6b191aa086b392bc3e52ed4ce0ed8d78e24b203e6c5e867560a05d1121 + languageName: node + linkType: hard + "fill-range@npm:^7.1.1": version: 7.1.1 resolution: "fill-range@npm:7.1.1" @@ -16291,20 +16359,18 @@ __metadata: languageName: node linkType: hard -"ioredis@npm:^5.4.1, ioredis@npm:^5.4.2": - version: 5.6.0 - resolution: "ioredis@npm:5.6.0" +"ioredis@npm:5.11.1, ioredis@npm:^5.11.1": + version: 5.11.1 + resolution: "ioredis@npm:5.11.1" dependencies: - "@ioredis/commands": "npm:^1.1.1" - cluster-key-slot: "npm:^1.1.0" - debug: "npm:^4.3.4" - denque: "npm:^2.1.0" - lodash.defaults: "npm:^4.2.0" - lodash.isarguments: "npm:^3.1.0" - redis-errors: "npm:^1.2.0" - redis-parser: "npm:^3.0.0" - standard-as-callback: "npm:^2.1.0" - checksum: 10c0/a885e5146640fc448706871290ef424ffa39af561f7ee3cf1590085209a509f85e99082bdaaf3cd32fa66758aea3fc2055d1109648ddca96fac4944bf2092c30 + "@ioredis/commands": "npm:1.10.0" + cluster-key-slot: "npm:1.1.1" + debug: "npm:4.4.3" + denque: "npm:2.1.0" + redis-errors: "npm:1.2.0" + redis-parser: "npm:3.0.0" + standard-as-callback: "npm:2.1.0" + checksum: 10c0/a8b27043cf2c045dfc93f40a32ce24cf9f8b57799a37f4234c4b925c365ccf131629590f94a512f546fda2ba8ed034009c94c4933ecd44c50bc166636d929fd6 languageName: node linkType: hard @@ -16776,6 +16842,19 @@ __metadata: languageName: node linkType: hard +"jake@npm:^10.8.5": + version: 10.9.4 + resolution: "jake@npm:10.9.4" + dependencies: + async: "npm:^3.2.6" + filelist: "npm:^1.0.4" + picocolors: "npm:^1.1.1" + bin: + jake: bin/cli.js + checksum: 10c0/bb52f000340d4a32f1a3893b9abe56ef2b77c25da4dbf2c0c874a8159d082dddda50a5ad10e26060198bd645b928ba8dba3b362710f46a247e335321188c5a9c + languageName: node + linkType: hard + "jiti@npm:2.4.2": version: 2.4.2 resolution: "jiti@npm:2.4.2" @@ -17445,20 +17524,6 @@ __metadata: languageName: node linkType: hard -"lodash.defaults@npm:^4.2.0": - version: 4.2.0 - resolution: "lodash.defaults@npm:4.2.0" - checksum: 10c0/d5b77aeb702caa69b17be1358faece33a84497bcca814897383c58b28a2f8dfc381b1d9edbec239f8b425126a3bbe4916223da2a576bb0411c2cefd67df80707 - languageName: node - linkType: hard - -"lodash.isarguments@npm:^3.1.0": - version: 3.1.0 - resolution: "lodash.isarguments@npm:3.1.0" - checksum: 10c0/5e8f95ba10975900a3920fb039a3f89a5a79359a1b5565e4e5b4310ed6ebe64011e31d402e34f577eca983a1fc01ff86c926e3cbe602e1ddfc858fdd353e62d8 - languageName: node - linkType: hard - "lodash.isplainobject@npm:^4.0.6": version: 4.0.6 resolution: "lodash.isplainobject@npm:4.0.6" @@ -17473,7 +17538,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.21": +"lodash@npm:^4.17.11, lodash@npm:^4.17.21": version: 4.18.1 resolution: "lodash@npm:4.18.1" checksum: 10c0/757228fc68805c59789e82185135cf85f05d0b2d3d54631d680ca79ec21944ec8314d4533639a14b8bcfbd97a517e78960933041a5af17ecb693ec6eecb99a27 @@ -18430,6 +18495,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^5.0.1": + version: 5.1.9 + resolution: "minimatch@npm:5.1.9" + dependencies: + brace-expansion: "npm:^2.0.1" + checksum: 10c0/4202718683815a7288b13e470160a4f9560cf392adef4f453927505817e01ef6b3476ecde13cfcaed17e7326dd3b69ad44eb2daeb19a217c5500f9277893f1d6 + languageName: node + linkType: hard + "minimatch@npm:^9.0.4, minimatch@npm:^9.0.5": version: 9.0.9 resolution: "minimatch@npm:9.0.9" @@ -18611,16 +18685,16 @@ __metadata: languageName: node linkType: hard -"msgpackr-extract@npm:^3.0.2": - version: 3.0.3 - resolution: "msgpackr-extract@npm:3.0.3" - dependencies: - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.3" +"msgpackr-extract@npm:^3.0.4": + version: 3.0.4 + resolution: "msgpackr-extract@npm:3.0.4" + dependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.4" node-gyp: "npm:latest" node-gyp-build-optional-packages: "npm:5.2.2" dependenciesMeta: @@ -18638,19 +18712,19 @@ __metadata: optional: true bin: download-msgpackr-prebuilds: bin/download-prebuilds.js - checksum: 10c0/e504fd8bf86a29d7527c83776530ee6dc92dcb0273bb3679fd4a85173efead7f0ee32fb82c8410a13c33ef32828c45f81118ffc0fbed5d6842e72299894623b4 + checksum: 10c0/582a9d17abbf3019e600e948736695056280ce401fd0235ee2474e95f9952208b9f6cce4d0e355b03b7d3c5630e6c3d11fe5fc27fdedb2311cce48de464338d8 languageName: node linkType: hard -"msgpackr@npm:^1.11.2": - version: 1.11.2 - resolution: "msgpackr@npm:1.11.2" +"msgpackr@npm:2.0.5": + version: 2.0.5 + resolution: "msgpackr@npm:2.0.5" dependencies: - msgpackr-extract: "npm:^3.0.2" + msgpackr-extract: "npm:^3.0.4" dependenciesMeta: msgpackr-extract: optional: true - checksum: 10c0/7d2e81ca82c397b2352d470d6bc8f4a967fe4fe14f8fc1fc9906b23009fdfb543999b1ad29c700b8861581e0b6bf903d6f0fefb69a09375cbca6d4d802e6c906 + checksum: 10c0/7ac9820cecd44d24d2ef07994405a277509f004d4fb0a77d04e10e9071d0599e86ad53a9daf673e801518230774bdaca72b1fdcff2b516584ab58bd19362e6ad languageName: node linkType: hard @@ -18862,7 +18936,7 @@ __metadata: languageName: node linkType: hard -"node-abort-controller@npm:^3.1.1": +"node-abort-controller@npm:3.1.1": version: 3.1.1 resolution: "node-abort-controller@npm:3.1.1" checksum: 10c0/f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 @@ -20870,14 +20944,23 @@ __metadata: languageName: node linkType: hard -"redis-errors@npm:^1.0.0, redis-errors@npm:^1.2.0": +"redis-errors@npm:1.2.0, redis-errors@npm:^1.0.0": version: 1.2.0 resolution: "redis-errors@npm:1.2.0" checksum: 10c0/5b316736e9f532d91a35bff631335137a4f974927bb2fb42bf8c2f18879173a211787db8ac4c3fde8f75ed6233eb0888e55d52510b5620e30d69d7d719c8b8a7 languageName: node linkType: hard -"redis-parser@npm:^3.0.0": +"redis-info@npm:^3.1.0": + version: 3.1.0 + resolution: "redis-info@npm:3.1.0" + dependencies: + lodash: "npm:^4.17.11" + checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38 + languageName: node + linkType: hard + +"redis-parser@npm:3.0.0": version: 3.0.0 resolution: "redis-parser@npm:3.0.0" dependencies: @@ -21567,6 +21650,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:7.8.5": + version: 7.8.5 + resolution: "semver@npm:7.8.5" + bin: + semver: bin/semver.js + checksum: 10c0/b1f3127a5be8125a94f37188b361c212466c292c6910adce3ec106cff5dc211ccaedc4739c11bb70fda59d6fc1f040a9bca289f4e093451521a2372e5231fe0c + languageName: node + linkType: hard + "semver@npm:^6.3.1": version: 6.3.1 resolution: "semver@npm:6.3.1" @@ -21576,7 +21668,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.3": +"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.6.0, semver@npm:^7.6.3": version: 7.7.1 resolution: "semver@npm:7.7.1" bin: @@ -22310,7 +22402,7 @@ __metadata: languageName: node linkType: hard -"standard-as-callback@npm:^2.1.0": +"standard-as-callback@npm:2.1.0": version: 2.1.0 resolution: "standard-as-callback@npm:2.1.0" checksum: 10c0/012677236e3d3fdc5689d29e64ea8a599331c4babe86956bf92fc5e127d53f85411c5536ee0079c52c43beb0026b5ce7aa1d834dd35dd026e82a15d1bcaead1f @@ -23241,7 +23333,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": +"tslib@npm:2.8.1, tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": version: 2.8.1 resolution: "tslib@npm:2.8.1" checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 From 81c419067d75120a068c99d0f44d40595b53e6e3 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 16:14:49 -0700 Subject: [PATCH 07/10] simplicity: remove much of the web changes --- packages/backend/src/jobManager.ts | 2 - .../migration.sql | 20 - .../migration.sql | 22 -- packages/db/prisma/schema.prisma | 65 +++- packages/db/tools/scripts/inject-repo-data.ts | 22 +- packages/shared/src/queue.ts | 28 -- packages/web/src/actions.ts | 38 +- .../components/defaultSidebar/index.tsx | 5 +- .../src/app/(app)/chat/chatLandingPage.tsx | 11 +- packages/web/src/app/(app)/repos/layout.tsx | 9 + .../search/components/searchLandingPage.tsx | 8 +- .../(app)/settings/connections/[id]/page.tsx | 211 +++++++++++ .../components/connectionJobsTable.tsx | 344 ++++++++++++++++++ .../components/connectionsTable.tsx | 294 +++++++++++++++ .../connections/connectionSyncLogsDialog.tsx | 203 ----------- .../settings/connections/connectionsList.tsx | 282 -------------- .../app/(app)/settings/connections/page.tsx | 97 ++--- .../web/src/app/(app)/settings/layout.tsx | 7 + .../web/src/features/workerApi/actions.ts | 75 +--- 19 files changed, 1069 insertions(+), 674 deletions(-) delete mode 100644 packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql delete mode 100644 packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql create mode 100644 packages/web/src/app/(app)/settings/connections/[id]/page.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx create mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx delete mode 100644 packages/web/src/app/(app)/settings/connections/connectionsList.tsx diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index b05ab4264..97c44da09 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -123,7 +123,6 @@ export class BullMQJobManager implements JobManager { `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, ); const lifecycleContext = this.jobLifecycleContext(job); - jobLogger.debug(`Started workload "${spec.name}"`); try { await workload.onStarted?.(lifecycleContext); @@ -134,7 +133,6 @@ export class BullMQJobManager implements JobManager { updateProgress: (progress) => job.updateProgress(progress), trigger: (target, data) => this.trigger(target, data), }); - jobLogger.debug(`Completed workload "${spec.name}"`); return result; } catch (error) { jobLogger.error(`Workload "${spec.name}" attempt failed`, error); diff --git a/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql b/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql deleted file mode 100644 index c4315cee2..000000000 --- a/packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* - Warnings: - - - You are about to drop the `ConnectionSyncJob` table. If the table is not empty, all the data it contains will be lost. - -*/ --- DropForeignKey -ALTER TABLE "ConnectionSyncJob" DROP CONSTRAINT "ConnectionSyncJob_connectionId_fkey"; - --- AlterTable -ALTER TABLE "Connection" ADD COLUMN "latestSyncJobId" TEXT; - --- DropTable -DROP TABLE "ConnectionSyncJob"; - --- DropEnum -DROP TYPE "ConnectionSyncJobStatus"; - --- DropEnum -DROP TYPE "ConnectionSyncStatus"; diff --git a/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql b/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql deleted file mode 100644 index ab019a779..000000000 --- a/packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* - Warnings: - - - You are about to drop the column `latestIndexingJobStatus` on the `Repo` table. All the data in the column will be lost. - - You are about to drop the `RepoIndexingJob` table. If the table is not empty, all the data it contains will be lost. - -*/ --- DropForeignKey -ALTER TABLE "RepoIndexingJob" DROP CONSTRAINT "RepoIndexingJob_repoId_fkey"; - --- AlterTable -ALTER TABLE "Repo" DROP COLUMN "latestIndexingJobStatus", -ADD COLUMN "latestIndexingJobId" TEXT; - --- DropTable -DROP TABLE "RepoIndexingJob"; - --- DropEnum -DROP TYPE "RepoIndexingJobStatus"; - --- DropEnum -DROP TYPE "RepoIndexingJobType"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 198b13a81..e43f2887b 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -10,6 +10,15 @@ datasource db { url = env("DATABASE_URL") } +enum ConnectionSyncStatus { + SYNC_NEEDED + IN_SYNC_QUEUE + SYNCING + SYNCED + SYNCED_WITH_WARNINGS + FAILED +} + enum ChatVisibility { PRIVATE PUBLIC @@ -65,10 +74,10 @@ model Repo { permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. + jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. - latestIndexingJobId String? - indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). + latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. pushedAt DateTime? /// The timestamp of the most recent commit across all branches. external_id String /// The id of the repo in the external service @@ -86,6 +95,35 @@ model Repo { @@index([indexedAt]) } +enum RepoIndexingJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +enum RepoIndexingJobType { + INDEX + CLEANUP +} + +model RepoIndexingJob { + id String @id @default(cuid()) + type RepoIndexingJobType + status RepoIndexingJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts + + errorMessage String? + + repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) + repoId Int + + @@index([repoId, type, status]) +} + enum RepoPermissionSyncJobStatus { PENDING IN_PROGRESS @@ -143,9 +181,9 @@ model Connection { // The type of connection (e.g., github, gitlab, etc.) connectionType ConnectionType + syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? - latestSyncJobId String? /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -170,6 +208,27 @@ model Connection { @@unique([name, orgId]) } +enum ConnectionSyncJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +model ConnectionSyncJob { + id String @id @default(cuid()) + status ConnectionSyncJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + + warningMessages String[] + errorMessage String? + + connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) + connectionId Int +} + model RepoToConnection { addedAt DateTime @default(now()) diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 5540f97b4..609bcdcc7 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -2,6 +2,7 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; const NUM_REPOS = 1000; +const NUM_INDEXING_JOBS_PER_REPO = 10000; const NUM_PERMISSION_JOBS_PER_REPO = 10000; export const injectRepoData: Script = { @@ -36,6 +37,8 @@ export const injectRepoData: Script = { console.log(`Creating ${NUM_REPOS} repos...`); const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const; + const indexingJobTypes = ['INDEX', 'CLEANUP'] as const; + for (let i = 0; i < NUM_REPOS; i++) { const repo = await prisma.repo.create({ data: { @@ -68,8 +71,23 @@ export const injectRepoData: Script = { } }); } + + for (let j = 0; j < NUM_INDEXING_JOBS_PER_REPO; j++) { + const status = statuses[Math.floor(Math.random() * statuses.length)]; + const type = indexingJobTypes[Math.floor(Math.random() * indexingJobTypes.length)]; + await prisma.repoIndexingJob.create({ + data: { + repoId: repo.id, + type, + status, + completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, + errorMessage: status === 'FAILED' ? 'Mock indexing error' : null, + metadata: {} + } + }); + } } - console.log(`Created ${NUM_REPOS} repos with associated permission jobs.`); + console.log(`Created ${NUM_REPOS} repos with associated jobs.`); } -}; +}; \ No newline at end of file diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 585d1e457..81e20c21f 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -26,20 +26,6 @@ export const CONNECTION_QUEUE: QueueSpec<'connection'> = { keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, - onEnqueued: async ({ - prisma, - data: { connectionId }, - jobId - }) => { - await prisma.connection.update({ - where: { - id: connectionId - }, - data: { - latestSyncJobId: jobId - } - }); - }, dedupKey: (data) => `connection:${data.connectionId}`, } @@ -62,20 +48,6 @@ export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, - onEnqueued: async ({ - prisma, - data: { repoId }, - jobId, - }) => { - await prisma.repo.update({ - where: { - id: repoId, - }, - data: { - latestIndexingJobId: jobId, - }, - }); - }, dedupKey: (data) => `repo:${data.repoId}`, }; diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 5db1dbdfc..c64979173 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, ServiceError } from "@/lib/serviceError"; import { sew } from "@/middleware/sew"; -import { OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { ConnectionSyncJobStatus, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; @@ -274,6 +274,42 @@ export const getReposStats = async () => sew(() => }) ) +export const getConnectionStats = async () => sew(() => + withAuth(async ({ org, prisma }) => { + const [ + numberOfConnections, + numberOfConnectionsWithFirstTimeSyncJobsInProgress, + ] = await Promise.all([ + prisma.connection.count({ + where: { + orgId: org.id, + } + }), + prisma.connection.count({ + where: { + orgId: org.id, + syncedAt: null, + syncJobs: { + some: { + status: { + in: [ + ConnectionSyncJobStatus.PENDING, + ConnectionSyncJobStatus.IN_PROGRESS, + ] + } + } + } + } + }) + ]); + + return { + numberOfConnections, + numberOfConnectionsWithFirstTimeSyncJobsInProgress, + }; + }) +); + export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuth(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx index c09bad644..a0ccd6269 100644 --- a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx +++ b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx @@ -2,6 +2,7 @@ import { cookies } from "next/headers"; import { auth } from "@/auth"; import { HOME_VIEW_COOKIE_NAME } from "@/lib/constants"; import { HomeView } from "@/hooks/useHomeView"; +import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; @@ -45,9 +46,11 @@ export async function DefaultSidebar() { if (!isOwner) { return false; } + const connectionStats = await getConnectionStats(); const joinRequests = await getOrgAccountRequests(); + const hasConnectionNotification = !isServiceError(connectionStats) && connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0; const hasJoinRequestNotification = !isServiceError(joinRequests) && joinRequests.length > 0; - return hasJoinRequestNotification; + return hasConnectionNotification || hasJoinRequestNotification; })(); return ( diff --git a/packages/web/src/app/(app)/chat/chatLandingPage.tsx b/packages/web/src/app/(app)/chat/chatLandingPage.tsx index fdebccf77..b78e28e10 100644 --- a/packages/web/src/app/(app)/chat/chatLandingPage.tsx +++ b/packages/web/src/app/(app)/chat/chatLandingPage.tsx @@ -1,4 +1,4 @@ -import { getRepos, getSearchContexts } from "@/actions"; +import { getRepos, getReposStats, getSearchContexts } from "@/actions"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; @@ -35,6 +35,8 @@ export async function ChatLandingPage() { take: 10, }); + const repoStats = await getReposStats(); + if (isServiceError(allRepos)) { throw new ServiceErrorException(allRepos); } @@ -47,6 +49,10 @@ export async function ChatLandingPage() { throw new ServiceErrorException(carouselRepos); } + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { try { return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; @@ -78,8 +84,7 @@ export async function ChatLandingPage() {
diff --git a/packages/web/src/app/(app)/repos/layout.tsx b/packages/web/src/app/(app)/repos/layout.tsx index 61f3cd34e..88738d22e 100644 --- a/packages/web/src/app/(app)/repos/layout.tsx +++ b/packages/web/src/app/(app)/repos/layout.tsx @@ -1,3 +1,7 @@ +import { getReposStats } from "@/actions"; +import { ServiceErrorException } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; + interface LayoutProps { children: React.ReactNode; } @@ -7,6 +11,11 @@ export default async function Layout( ) { const { children } = props; + const repoStats = await getReposStats(); + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + return (
diff --git a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx index a6d072f29..e9236607e 100644 --- a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx +++ b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx @@ -5,7 +5,7 @@ import { SyntaxReferenceGuideHint } from "../../components/syntaxReferenceGuideH import Link from "next/link" import { SearchBar } from "../../components/searchBar" import { SearchModeSelector } from "../../components/searchModeSelector" -import { getRepos } from "@/actions" +import { getRepos, getReposStats } from "@/actions" import { ServiceErrorException } from "@/lib/serviceError" import { isServiceError } from "@/lib/utils" @@ -25,7 +25,10 @@ export const SearchLandingPage = async ({ take: 10, }); + const repoStats = await getReposStats(); + if (isServiceError(carouselRepos)) throw new ServiceErrorException(carouselRepos); + if (isServiceError(repoStats)) throw new ServiceErrorException(repoStats); return (
@@ -52,8 +55,7 @@ export const SearchLandingPage = async ({
diff --git a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx new file mode 100644 index 000000000..edcc61069 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx @@ -0,0 +1,211 @@ +import { sew } from "@/middleware/sew"; +import { BackButton } from "@/app/(app)/components/backButton"; +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { notFound as notFoundServiceError, ServiceErrorException } from "@/lib/serviceError"; +import { notFound } from "next/navigation"; +import { isServiceError } from "@/lib/utils"; +import { withAuth } from "@/middleware/withAuth"; +import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { env, getConfigSettings } from "@sourcebot/shared"; +import { Info } from "lucide-react"; +import Link from "next/link"; +import { Suspense } from "react"; +import { ConnectionJobsTable } from "../components/connectionJobsTable"; + +interface ConnectionDetailPageProps { + params: Promise<{ + id: string + }> +} + +export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { + const params = await props.params; + const { id } = params; + + const connectionId = Number.parseInt(id); + if (isNaN(connectionId)) { + return notFound(); + } + + const connection = await getConnectionWithJobs(connectionId); + if (isServiceError(connection)) { + throw new ServiceErrorException(connection); + } + + const configSettings = await getConfigSettings(env.CONFIG_PATH); + + const nextSyncAttempt = (() => { + const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null; + if (!latestJob) { + return undefined; + } + + if (latestJob.completedAt) { + return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs); + } + + return undefined; + })(); + + // Extracts the code host URL from the connection config. + const codeHostUrl: string = (() => { + const connectionType = connection.connectionType; + switch (connectionType) { + case 'github': { + const config = connection.config as unknown as GithubConnectionConfig; + return config.url ?? 'https://github.com'; + } + case 'gitlab': { + const config = connection.config as unknown as GitlabConnectionConfig; + return config.url ?? 'https://gitlab.com'; + } + case 'gitea': { + const config = connection.config as unknown as GiteaConnectionConfig; + return config.url ?? 'https://gitea.com'; + } + case 'gerrit': { + const config = connection.config as unknown as GerritConnectionConfig; + return config.url; + } + case 'bitbucket': { + const config = connection.config as unknown as BitbucketConnectionConfig; + if (config.deploymentType === 'cloud') { + return config.url ?? 'https://bitbucket.org'; + } else { + return config.url!; + } + } + case 'azuredevops': { + const config = connection.config as unknown as AzureDevOpsConnectionConfig; + return config.url ?? 'https://dev.azure.com'; + } + case 'git': { + const config = connection.config as unknown as GenericGitHostConnectionConfig; + return config.url; + } + } + })(); + + return ( +
+ +
+

{connection.name}

+ + + {codeHostUrl} + +
+ +
+ + + + Created + + + + + +

When this connection was first added to Sourcebot

+
+
+
+
+ + + +
+ + + + + Last synced + + + + + +

The last time this connection was successfully synced

+
+
+
+
+ + {connection.syncedAt ? : "Never"} + +
+ + + + + Scheduled + + + + + +

When the connection will be resynced next. Modifying the config will also trigger a resync.

+
+
+
+
+ + {nextSyncAttempt ? : "-"} + +
+
+ + + + Sync History + History of all sync jobs for this connection. + + + }> + + + + +
+ ) +} + +const getConnectionWithJobs = async (id: number) => sew(() => + withAuth(async ({ prisma, org }) => { + const connection = await prisma.connection.findUnique({ + where: { + id, + orgId: org.id, + }, + include: { + syncJobs: { + orderBy: { + createdAt: 'desc', + }, + }, + }, + }); + + if (!connection) { + return notFoundServiceError(); + } + + return connection; + }) +) \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx new file mode 100644 index 000000000..9277d81b7 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx @@ -0,0 +1,344 @@ +"use client" + +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" +import { + type ColumnDef, + type ColumnFiltersState, + type SortingState, + type VisibilityState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table" +import { cva } from "class-variance-authority" +import { AlertCircle, AlertTriangle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" +import * as React from "react" +import { CopyIconButton } from "@/app/(app)/components/copyIconButton" +import { useMemo } from "react" +import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter" +import { useRouter } from "next/navigation" +import { useToast } from "@/components/hooks/use-toast" +import { DisplayDate } from "@/app/(app)/components/DisplayDate" +import { LoadingButton } from "@/components/ui/loading-button" +import { syncConnection } from "@/features/workerApi/actions" +import { isServiceError } from "@/lib/utils" + + +export type ConnectionSyncJob = { + id: string + status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" + createdAt: Date + updatedAt: Date + completedAt: Date | null + errorMessage: string | null + warningMessages: string[] +} + +const statusBadgeVariants = cva("", { + variants: { + status: { + PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", + IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", + COMPLETED: "bg-green-600 text-white hover:bg-green-700", + FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + }, + }, +}) + +const getStatusBadge = (status: ConnectionSyncJob["status"]) => { + const labels = { + PENDING: "Pending", + IN_PROGRESS: "In Progress", + COMPLETED: "Completed", + FAILED: "Failed", + } + + return {labels[status]} +} + +const getDuration = (start: Date, end: Date | null) => { + if (!end) return "-" + const diff = end.getTime() - start.getTime() + const minutes = Math.floor(diff / 60000) + const seconds = Math.floor((diff % 60000) / 1000) + return `${minutes}m ${seconds}s` +} + +export const columns: ColumnDef[] = [ + { + accessorKey: "status", + header: "Status", + cell: ({ row }) => { + const job = row.original + return ( +
+ {getStatusBadge(row.getValue("status"))} + {job.errorMessage ? ( + + + + + + + + {job.errorMessage} + + + + + ) : job.warningMessages.length > 0 ? ( + + + + + + +

{job.warningMessages.length} warning(s) while syncing:

+
+ {job.warningMessages.map((warning, index) => ( +
+ {index + 1}. + {warning} +
+ ))} +
+
+
+
+ ) : null} +
+ ) + }, + filterFn: (row, id, value) => { + return value.includes(row.getValue(id)) + }, + }, + { + accessorKey: "createdAt", + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => , + }, + { + accessorKey: "completedAt", + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const completedAt = row.getValue("completedAt") as Date | null; + if (!completedAt) { + return "-"; + } + + return + }, + }, + { + id: "duration", + header: "Duration", + cell: ({ row }) => { + const job = row.original + return getDuration(job.createdAt, job.completedAt) + }, + }, + { + accessorKey: "id", + header: "Job ID", + cell: ({ row }) => { + const id = row.getValue("id") as string + return ( +
+ {id} + { + navigator.clipboard.writeText(id); + return true; + }} /> +
+ ) + }, + }, +] + +export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSyncJob[], connectionId: number }) => { + const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) + const [columnFilters, setColumnFilters] = React.useState([]) + const [columnVisibility, setColumnVisibility] = React.useState({}) + const router = useRouter(); + const { toast } = useToast(); + + const [isSyncSubmitting, setIsSyncSubmitting] = React.useState(false); + const onSyncButtonClick = React.useCallback(async () => { + setIsSyncSubmitting(true); + const response = await syncConnection(connectionId); + + if (!isServiceError(response)) { + const { jobId } = response; + toast({ + description: `✅ Connection synced successfully. Job ID: ${jobId}`, + }) + router.refresh(); + } else { + toast({ + description: `❌ Failed to sync connection. ${response.message}`, + }); + } + + setIsSyncSubmitting(false); + }, [connectionId, router, toast]); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + state: { + sorting, + columnFilters, + columnVisibility, + }, + }) + + const { + numCompleted, + numInProgress, + numPending, + numFailed, + } = useMemo(() => { + return { + numCompleted: data.filter((job) => job.status === "COMPLETED").length, + numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, + numPending: data.filter((job) => job.status === "PENDING").length, + numFailed: data.filter((job) => job.status === "FAILED").length, + }; + }, [data]); + + return ( +
+
+ + +
+ + + + + Trigger sync + +
+
+ +
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + + ) + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + {flexRender(cell.column.columnDef.cell, cell.getContext())} + ))} + + )) + ) : ( + + + No sync jobs found. + + + )} + +
+
+ +
+
+ {table.getFilteredRowModel().rows.length} job(s) total +
+
+ + +
+
+
+ ) +} diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx new file mode 100644 index 000000000..e52e7ef1e --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx @@ -0,0 +1,294 @@ +"use client" + +import { DisplayDate } from "@/app/(app)/components/DisplayDate" +import { NotificationDot } from "@/app/(app)/components/notificationDot" +import { useToast } from "@/components/hooks/use-toast" +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { Input } from "@/components/ui/input" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" +import { getCodeHostIcon } from "@/lib/utils" +import { ConnectionType } from "@sourcebot/db" +import { + type ColumnDef, + type ColumnFiltersState, + type SortingState, + type VisibilityState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table" +import { cva } from "class-variance-authority" +import { ArrowUpDown, RefreshCwIcon } from "lucide-react" +import Image from "next/image" +import Link from "next/link" +import { useRouter } from "next/navigation" +import { useMemo, useState } from "react" + + +export type Connection = { + id: number + name: string + syncedAt: Date | null + connectionType: ConnectionType + latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null + isFirstTimeSync: boolean +} + +const statusBadgeVariants = cva("", { + variants: { + status: { + PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", + IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", + COMPLETED: "bg-green-600 text-white hover:bg-green-700", + FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + }, + }, +}) + +const getStatusBadge = (status: Connection["latestJobStatus"]) => { + if (!status) { + return "-"; + } + + const labels = { + PENDING: "Pending", + IN_PROGRESS: "In Progress", + COMPLETED: "Completed", + FAILED: "Failed", + } + + return {labels[status]} +} + +export const columns: ColumnDef[] = [ + { + accessorKey: "name", + size: 400, + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const connection = row.original; + const codeHostIcon = getCodeHostIcon(connection.connectionType); + + return ( +
+ {`${connection.connectionType} + + {connection.name} + + {connection.isFirstTimeSync && ( + + + + + + + + This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete. + + + )} +
+ ) + }, + }, + { + accessorKey: "latestJobStatus", + size: 150, + header: "Lastest status", + cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), + }, + { + accessorKey: "syncedAt", + size: 200, + header: ({ column }) => { + return ( + + ) + }, + cell: ({ row }) => { + const syncedAt = row.getValue("syncedAt") as Date | null; + if (!syncedAt) { + return "-"; + } + + return ( + + ) + } + }, +] + +export const ConnectionsTable = ({ data }: { data: Connection[] }) => { + const [sorting, setSorting] = useState([]) + const [columnFilters, setColumnFilters] = useState([]) + const [columnVisibility, setColumnVisibility] = useState({}) + const [rowSelection, setRowSelection] = useState({}) + const router = useRouter(); + const { toast } = useToast(); + + const { + numCompleted, + numInProgress, + numPending, + numFailed, + numNoJobs, + } = useMemo(() => { + return { + numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length, + numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length, + numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length, + numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length, + numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length, + } + }, [data]); + + const table = useReactTable({ + data, + columns, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + onRowSelectionChange: setRowSelection, + columnResizeMode: 'onChange', + enableColumnResizing: false, + state: { + sorting, + columnFilters, + columnVisibility, + rowSelection, + }, + }) + + return ( +
+
+ table.getColumn("name")?.setFilterValue(event.target.value)} + className="max-w-sm" + /> + + +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => { + return ( + + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + + ) + })} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ))} + + )) + ) : ( + + + No results. + + + )} + +
+
+
+
+ {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total +
+
+ + +
+
+
+ ) +} diff --git a/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx b/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx deleted file mode 100644 index 0c41c6b38..000000000 --- a/packages/web/src/app/(app)/settings/connections/connectionSyncLogsDialog.tsx +++ /dev/null @@ -1,203 +0,0 @@ -"use client"; - -import { Button } from "@/components/ui/button"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; -import { getConnectionSyncJobLogs } from "@/features/workerApi/actions"; -import { cn, isServiceError } from "@/lib/utils"; -import type { JobLogEntry, JobLogLevel } from "@sourcebot/shared"; -import { Loader2, RefreshCw } from "lucide-react"; -import { useEffect, useRef, useState } from "react"; - -const LOG_POLL_INTERVAL_MS = 1000; - -const JOB_LOG_LEVEL_CLASS_NAMES: Record = { - debug: "text-muted-foreground", - info: "text-blue-600 dark:text-blue-400", - warn: "text-amber-600 dark:text-amber-400", - error: "text-destructive", -}; - -const formatLogTimestamp = (timestamp: string | null) => { - if (!timestamp) { - return "—"; - } - return new Date(timestamp).toLocaleTimeString(); -}; - -type ConnectionSyncLogsDialogProps = { - connectionId: number; - connectionName: string; - jobId: string | null; - open: boolean; - onOpenChange: (open: boolean) => void; -}; - -export const ConnectionSyncLogsDialog = ({ - connectionId, - connectionName, - jobId, - open, - onOpenChange, -}: ConnectionSyncLogsDialogProps) => { - const [logs, setLogs] = useState([]); - const [hasLoaded, setHasLoaded] = useState(false); - const [isRefreshing, setIsRefreshing] = useState(false); - const [logsError, setLogsError] = useState(null); - const pollNowRef = useRef<() => void>(() => undefined); - - useEffect(() => { - if (!open || !jobId) { - return; - } - - let cursor = 0; - let isCancelled = false; - let isRequestInFlight = false; - - setLogs([]); - setHasLoaded(false); - setLogsError(null); - - const poll = async () => { - if (isRequestInFlight) { - return; - } - - isRequestInFlight = true; - setIsRefreshing(true); - - try { - const result = await getConnectionSyncJobLogs( - connectionId, - jobId, - cursor, - ); - if (isCancelled) { - return; - } - if (isServiceError(result)) { - setLogsError(result.message); - return; - } - - setLogsError(null); - setLogs((currentLogs) => ( - result.count < cursor - ? result.logs - : [...currentLogs, ...result.logs] - )); - cursor = result.count; - } catch { - if (!isCancelled) { - setLogsError("Failed to load logs for this sync."); - } - } finally { - if (!isCancelled) { - setHasLoaded(true); - setIsRefreshing(false); - } - isRequestInFlight = false; - } - }; - - const pollNow = () => { - void poll(); - }; - pollNowRef.current = pollNow; - - pollNow(); - const interval = window.setInterval(pollNow, LOG_POLL_INTERVAL_MS); - - return () => { - isCancelled = true; - window.clearInterval(interval); - if (pollNowRef.current === pollNow) { - pollNowRef.current = () => undefined; - } - }; - }, [connectionId, jobId, open]); - - const isInitialLoading = !hasLoaded && isRefreshing; - - return ( - - -
- - {connectionName} sync logs - - Latest sync job {jobId}. Logs update automatically while this dialog is open. - - - -
- -
- {isInitialLoading ? ( -
- - Loading logs… -
- ) : logs.length > 0 ? ( - <> - {logsError && ( -
- {logsError} -
- )} -
- {logs.map((entry, index) => ( -
- - {formatLogTimestamp(entry.timestamp)} - - - {entry.level} - -

{entry.message}

-
- ))} -
- - ) : logsError ? ( -
- {logsError} -
- ) : ( -
- No logs were recorded for this job. -
- )} -
-
-
- ); -}; diff --git a/packages/web/src/app/(app)/settings/connections/connectionsList.tsx b/packages/web/src/app/(app)/settings/connections/connectionsList.tsx deleted file mode 100644 index 49acc679b..000000000 --- a/packages/web/src/app/(app)/settings/connections/connectionsList.tsx +++ /dev/null @@ -1,282 +0,0 @@ -"use client"; - -import { DisplayDate } from "@/app/(app)/components/DisplayDate"; -import { useToast } from "@/components/hooks/use-toast"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import { Input } from "@/components/ui/input"; -import { syncConnection } from "@/features/workerApi/actions"; -import { cn, getCodeHostIcon, isServiceError } from "@/lib/utils"; -import { ConnectionType } from "@sourcebot/db"; -import { AlertCircle, CheckCircle2, CircleDashed, FileText, MoreHorizontal, RefreshCw, Search } from "lucide-react"; -import Image from "next/image"; -import { useRouter } from "next/navigation"; -import { useEffect, useMemo, useState } from "react"; -import { ConnectionSyncLogsDialog } from "./connectionSyncLogsDialog"; -import { WorkloadJob } from "@sourcebot/shared"; - -const SYNC_STATUS_POLL_INTERVAL_MS = 1000; - -export type ConnectionV2 = { - id: number; - name: string; - connectionType: ConnectionType; - syncedAt: Date | null; - currentJob: WorkloadJob<'connection'> | null; -}; - -const getConnectionStatus = (connection: ConnectionV2) => { - switch (connection.currentJob?.status) { - case "PENDING": - return { - key: "syncing", - label: "Queued", - icon: , - className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", - }; - case "IN_PROGRESS": - return { - key: "syncing", - label: "Syncing", - icon: , - className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", - }; - case "FAILED": - return { - key: "failed", - label: "Sync failed", - icon: , - className: "border-destructive/30 bg-destructive/10 text-destructive", - }; - case "COMPLETED": - return { - key: "healthy", - label: "Healthy", - icon: , - className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", - }; - default: - return connection.syncedAt - ? { - key: "healthy", - label: "Healthy", - icon: , - className: "border-green-200 bg-green-50 text-green-700 dark:border-green-900 dark:bg-green-950 dark:text-green-300", - } - : { - key: "not-synced", - label: "Never synced", - icon: , - className: "border-border bg-muted text-muted-foreground", - }; - } -}; - -const ConnectionRow = ({ connection }: { connection: ConnectionV2 }) => { - const [isSubmitting, setIsSubmitting] = useState(false); - const [submittedJobId, setSubmittedJobId] = useState(null); - const [isLogsOpen, setIsLogsOpen] = useState(false); - const router = useRouter(); - const { toast } = useToast(); - const codeHostIcon = getCodeHostIcon(connection.connectionType); - const isRunning = connection.currentJob?.status === "PENDING" || - connection.currentJob?.status === "IN_PROGRESS"; - const isSubmittedJobSettled = connection.currentJob?.id === submittedJobId && - (connection.currentJob.status === "COMPLETED" || connection.currentJob.status === "FAILED"); - const isSyncRunning = isSubmitting || isRunning || (submittedJobId !== null && !isSubmittedJobSettled); - const status = isSyncRunning && !isRunning - ? { - key: "syncing", - label: "Syncing", - icon: , - className: "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-900 dark:bg-blue-950 dark:text-blue-300", - } - : getConnectionStatus(connection); - - useEffect(() => { - if (!isSyncRunning) { - return; - } - - const interval = window.setInterval(() => { - router.refresh(); - }, SYNC_STATUS_POLL_INTERVAL_MS); - - return () => { - window.clearInterval(interval); - }; - }, [isSyncRunning, router]); - - const onSync = async () => { - setIsSubmitting(true); - try { - const result = await syncConnection(connection.id); - if (isServiceError(result)) { - toast({ - description: `❌ Failed to sync connection. ${result.message}`, - }); - return; - } - - setSubmittedJobId(result.jobId); - toast({ - description: `✅ Connection sync triggered. Job ID: ${result.jobId}`, - }); - router.refresh(); - } catch { - toast({ - description: "❌ Failed to sync connection.", - }); - } finally { - setIsSubmitting(false); - } - }; - - return ( - <> -
-
-
- {`${connection.connectionType} -
-
-

{connection.name}

-

{connection.connectionType}

-
-
- -
-

Current status

- - {status.icon} - {status.label} - - {connection.currentJob?.status === "FAILED" && connection.currentJob.errorMessage && ( -

{connection.currentJob.errorMessage}

- )} -
- -
-

Last successful sync

-

- {connection.syncedAt - ? - : Not yet synced} -

-
- - - - - - - { - setIsLogsOpen(true); - }} - > - - View logs - - { - void onSync(); - }} - > - - {isSyncRunning ? "Sync in progress" : "Trigger sync"} - - - -
- - - - ); -}; - -export const ConnectionsList = ({ data }: { data: ConnectionV2[] }) => { - const [query, setQuery] = useState(""); - const healthyCount = useMemo( - () => data.filter((connection) => getConnectionStatus(connection).key === "healthy").length, - [data], - ); - const filteredConnections = useMemo(() => { - const normalizedQuery = query.trim().toLowerCase(); - - return data.filter((connection) => ( - normalizedQuery.length === 0 || - connection.name.toLowerCase().includes(normalizedQuery) || - connection.connectionType.toLowerCase().includes(normalizedQuery) - )); - }, [data, query]); - - return ( -
-
-
- - setQuery(event.target.value)} - placeholder="Search connections..." - className="pl-9" - /> -
-

- {healthyCount} healthy · {data.length} total -

-
- -
- {filteredConnections.length > 0 ? filteredConnections.map((connection, index) => ( -
0 ? "border-t" : undefined}> - -
- )) : ( -
-

{data.length === 0 ? "No code host connections" : "No matching connections"}

-

- {data.length === 0 - ? "Add a connection to begin syncing repositories." - : "Try changing your search."} -

-
- )} -
- - {filteredConnections.length > 0 && ( -

- Showing {filteredConnections.length} of {data.length} {data.length === 1 ? "connection" : "connections"} -

- )} -
- ); -}; diff --git a/packages/web/src/app/(app)/settings/connections/page.tsx b/packages/web/src/app/(app)/settings/connections/page.tsx index a1d99cd38..fdce7b08f 100644 --- a/packages/web/src/app/(app)/settings/connections/page.tsx +++ b/packages/web/src/app/(app)/settings/connections/page.tsx @@ -1,72 +1,77 @@ +import { sew } from "@/middleware/sew"; import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; -import { sew } from "@/middleware/sew"; import { withAuth } from "@/middleware/withAuth"; -import { getBullMQClient } from "@/lib/bullmqClient"; import Link from "next/link"; -import { ConnectionsList } from "./connectionsList"; -import { CONNECTION_QUEUE } from "@sourcebot/shared"; +import { ConnectionsTable } from "./components/connectionsTable"; +import { ConnectionSyncJobStatus } from "@prisma/client"; const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; -export default async function ConnectionsV2Page() { - const connections = await getConnectionsWithCurrentStatus(); - if (isServiceError(connections)) { - throw new ServiceErrorException(connections); +export default async function ConnectionsPage() { + const _connections = await getConnectionsWithLatestJob(); + if (isServiceError(_connections)) { + throw new ServiceErrorException(_connections); } + // Sort connections so that first time syncs are at the top. + const connections = _connections + .map((connection) => ({ + ...connection, + isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0, + latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null, + })) + .sort((a, b) => { + if (a.isFirstTimeSync && !b.isFirstTimeSync) { + return -1; + } + if (!a.isFirstTimeSync && b.isFirstTimeSync) { + return 1; + } + return a.name.localeCompare(b.name); + }); + return (
-
-

Code Host Connections

- - Prototype - -
-

- Monitor and sync your external code hosts.{" "} - - Learn more - -

+

Code Host Connections

+

Manage your connections to external code hosts. Learn more

- - + ({ + id: connection.id, + name: connection.name, + connectionType: connection.connectionType, + syncedAt: connection.syncedAt, + latestJobStatus: connection.latestJobStatus, + isFirstTimeSync: connection.isFirstTimeSync, + }))} />
- ); + ) } -const getConnectionsWithCurrentStatus = async () => sew(() => +const getConnectionsWithLatestJob = async () => sew(() => withAuth(async ({ prisma, org }) => { const connections = await prisma.connection.findMany({ where: { orgId: org.id, }, - select: { - id: true, - name: true, - connectionType: true, - syncedAt: true, - latestSyncJobId: true, + include: { + _count: { + select: { + syncJobs: true, + } + }, + syncJobs: { + orderBy: { + createdAt: 'desc' + }, + take: 1 + }, }, orderBy: { - name: 'asc', + name: 'asc' }, }); - return Promise.all(connections.map(async ({ - latestSyncJobId, - ...connection - }) => { - const job = latestSyncJobId - ? await getBullMQClient().getJob(CONNECTION_QUEUE, latestSyncJobId) - : null; - - return { - ...connection, - currentJob: job - }; - })); - }) -); + return connections; + })); \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/layout.tsx b/packages/web/src/app/(app)/settings/layout.tsx index e57e4180e..655ca28f5 100644 --- a/packages/web/src/app/(app)/settings/layout.tsx +++ b/packages/web/src/app/(app)/settings/layout.tsx @@ -3,6 +3,7 @@ import { Metadata } from "next" import { redirect } from "next/navigation"; import { auth } from "@/auth"; import { isServiceError } from "@/lib/utils"; +import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { OrgRole } from "@prisma/client"; @@ -47,6 +48,10 @@ export const getSidebarNavGroups = async () => numJoinRequests = requests.length; } + const connectionStats = await getConnectionStats(); + if (isServiceError(connectionStats)) { + throw new ServiceErrorException(connectionStats); + } const hasAskEntitlement = await hasEntitlement("ask"); const groups: NavGroup[] = [ @@ -114,6 +119,8 @@ export const getSidebarNavGroups = async () => { title: "Connections", href: `/settings/connections`, + hrefRegex: `/settings/connections(/[^/]+)?$`, + isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0, icon: "plug" as const, }, { diff --git a/packages/web/src/features/workerApi/actions.ts b/packages/web/src/features/workerApi/actions.ts index 906e28ac6..8492a7b7e 100644 --- a/packages/web/src/features/workerApi/actions.ts +++ b/packages/web/src/features/workerApi/actions.ts @@ -1,79 +1,38 @@ 'use server'; import { sew } from "@/middleware/sew"; -import { notFound, repositoryNotFound, unexpectedError } from "@/lib/serviceError"; +import { repositoryNotFound, unexpectedError } from "@/lib/serviceError"; import { withAuth, withOptionalAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; import { OrgRole } from "@sourcebot/db"; -import { CONNECTION_QUEUE, env } from "@sourcebot/shared"; +import { env } from "@sourcebot/shared"; import z from "zod"; -import { getBullMQClient } from "@/lib/bullmqClient"; +import { requestAccountPermissionSync } from "./client.server"; const WORKER_API_URL = env.WORKER_API_URL; export const syncConnection = async (connectionId: number) => sew(() => - withAuth(({ org, prisma, role }) => - withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const connection = await prisma.connection.findUnique({ - where: { - id: connectionId, - orgId: org.id, - }, - select: { - id: true, - orgId: true, - }, - }); - - if (!connection) { - return notFound('Connection not found'); - } - - const jobId = await getBullMQClient().enqueue(CONNECTION_QUEUE, { - connectionId: connection.id, - orgId: connection.orgId, - }); - - return { jobId }; - }) - ) -); - -export const getConnectionSyncJobLogs = async ( - connectionId: number, - jobId: string, - start = 0, -) => sew(() => - withAuth(({ org, prisma, role }) => + withAuth(({ role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { - const connection = await prisma.connection.findUnique({ - where: { - id: connectionId, - orgId: org.id, - }, - select: { - id: true, + const response = await fetch(`${WORKER_API_URL}/api/sync-connection`, { + method: 'POST', + body: JSON.stringify({ + connectionId + }), + headers: { + 'Content-Type': 'application/json', }, }); - if (!connection) { - return notFound('Connection not found'); - } - - const client = getBullMQClient(); - const job = await client.getJob(CONNECTION_QUEUE, jobId); - if ( - !job || - job.data.connectionId !== connection.id || - job.data.orgId !== org.id - ) { - return notFound('Connection sync job not found'); + if (!response.ok) { + return unexpectedError('Failed to sync connection'); } - return client.getJobLogs(CONNECTION_QUEUE, jobId, { - start: Number.isInteger(start) && start >= 0 ? start : 0, - ascending: true, + const data = await response.json(); + const schema = z.object({ + jobId: z.string(), }); + return schema.parse(data); }) ) ); From f6ad4b38d0444f10cdf331b0dfcefd8fc1f154fb Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 16:20:24 -0700 Subject: [PATCH 08/10] wip --- docs/snippets/schemas/v3/index.schema.mdx | 6 ++---- packages/backend/src/connectionWorkload.ts | 24 ++++++++++++++-------- packages/backend/src/index.ts | 6 +++++- packages/backend/src/repoIndexWorkload.ts | 4 ++-- packages/schemas/src/v3/index.schema.ts | 6 ++---- packages/schemas/src/v3/index.type.ts | 1 - schemas/v3/index.json | 3 +-- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 43367fe87..36e908e30 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -43,8 +43,7 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -230,8 +229,7 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index b05b9537e..9c35951cc 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -1,14 +1,22 @@ -import { Workload } from "./types.js"; -import { prisma } from "./prisma.js"; +import { Settings, Workload } from "./types.js"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; +import { PrismaClient } from "@sourcebot/db"; -export const connectionWorkload: Workload<'connection'> = { +interface Props { + db: PrismaClient, + settings: Settings; +} + +export const createConnectionWorkload = ({ + db, + settings +}: Props): Workload<'connection'> => ({ queueSpec: CONNECTION_QUEUE, - concurrency: 2, + concurrency: settings.maxConnectionSyncJobConcurrency, process: async ({ data: { connectionId, @@ -21,7 +29,7 @@ export const connectionWorkload: Workload<'connection'> = { connectionId, orgId, }); - const connection = await prisma.connection.findUniqueOrThrow({ + const connection = await db.connection.findUniqueOrThrow({ where: { id: connectionId } @@ -55,7 +63,7 @@ export const connectionWorkload: Workload<'connection'> = { // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't // appear in the repoData array above, and so the RepoToConnection record won't be re-created. // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await prisma.$transaction(async (tx) => { + await db.$transaction(async (tx) => { const deleteStart = performance.now(); await tx.connection.update({ where: { @@ -104,7 +112,7 @@ export const connectionWorkload: Workload<'connection'> = { }); }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); - await prisma.connection.update({ + await db.connection.update({ where: { id: connectionId, }, @@ -131,7 +139,7 @@ export const connectionWorkload: Workload<'connection'> = { connectionId, }); } -} +}) const discoverConnectionRepositories = async ({ config, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 5d6d196cf..8e3fbce99 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -13,7 +13,7 @@ import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; -import { connectionWorkload } from "./connectionWorkload.js"; +import { connectionWorkload, createConnectionWorkload } from "./connectionWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; @@ -51,6 +51,10 @@ const reconciliationWorkload = createReconciliationWorkload({ db: prisma, settings, }); +const connectionWorkload = createConnectionWorkload({ + db: prisma, + settings, +}); const repoIndexWorkload = createRepoIndexWorkload({ db: prisma, settings, diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index a2a66e77d..3377d1c7e 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -13,7 +13,7 @@ import { cleanupTempShards, indexGitRepository } from './zoekt.js'; const LOG_TAG = 'repo-index-workload'; const logger = createLogger(LOG_TAG); -interface RepoIndexWorkloadDependencies { +interface Props { db: PrismaClient; settings: Settings; } @@ -21,7 +21,7 @@ interface RepoIndexWorkloadDependencies { export const createRepoIndexWorkload = ({ db, settings, -}: RepoIndexWorkloadDependencies): Workload<'repo-index'> => ({ +}: Props): Workload<'repo-index'> => ({ queueSpec: REPO_INDEX_QUEUE, concurrency: settings.maxRepoIndexingJobConcurrency, process: async ({ data, logger: jobLogger, signal }) => { diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index ef5c461dd..15296f33a 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -42,8 +42,7 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", @@ -229,8 +228,7 @@ const schema = { "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index b3b03b34d..901e7762b 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -110,7 +110,6 @@ export interface Settings { */ reindexRepoPollingIntervalMs?: number; /** - * @deprecated * The number of connection sync jobs to run concurrently. Defaults to 2. */ maxConnectionSyncJobConcurrency?: number; diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 887d40eb7..6b5b4fbb1 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -41,8 +41,7 @@ "maxConnectionSyncJobConcurrency": { "type": "number", "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", - "minimum": 1, - "deprecated": true + "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", From d367d9faf8b7c1002d6aa9877b439f8b24b75bd7 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Fri, 31 Jul 2026 16:44:39 -0700 Subject: [PATCH 09/10] add necessary lifecyle hooks --- packages/backend/src/configManager.ts | 2 +- .../backend/src/connectionWorkload.test.ts | 125 +++++++++++++--- packages/backend/src/connectionWorkload.ts | 65 +++++++- packages/backend/src/index.ts | 2 +- packages/backend/src/jobManager.test.ts | 10 +- .../src/reconciliationWorkload.test.ts | 8 +- .../backend/src/reconciliationWorkload.ts | 2 +- .../backend/src/repoIndexWorkload.test.ts | 140 ++++++++++++++++++ packages/backend/src/repoIndexWorkload.ts | 80 +++++++++- packages/shared/src/bullmqClient.test.ts | 52 +++++-- packages/shared/src/queue.ts | 47 ++++-- 11 files changed, 468 insertions(+), 65 deletions(-) create mode 100644 packages/backend/src/repoIndexWorkload.test.ts diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 669d3d43e..8d04778a5 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -101,7 +101,7 @@ export class ConfigManager { if (connectionNeedsSyncing) { logger.debug(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`); - await this.jobManager.trigger('connection', { + await this.jobManager.trigger('connection-sync', { connectionId: connection.id, orgId: SINGLE_TENANT_ORG_ID, }) diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index d21d4a78a..679bd891b 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -1,8 +1,11 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; +import type { PrismaClient } from '@sourcebot/db'; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), + connectionSyncJobUpsert: vi.fn(), + connectionSyncJobUpdate: vi.fn(), transactionConnectionUpdate: vi.fn(), transactionRepoUpsert: vi.fn(), compileGithubConfig: vi.fn(), @@ -16,7 +19,7 @@ vi.mock('@sentry/node', () => ({ vi.mock('@sourcebot/shared', () => ({ CONNECTION_QUEUE: { - name: 'connection', + name: 'connection-sync', dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, jobOptions: { attempts: 2, @@ -36,23 +39,6 @@ vi.mock('@sourcebot/shared', () => ({ loadConfig: mocks.loadConfig, })); -vi.mock('./prisma.js', () => ({ - prisma: { - connection: { - findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, - update: mocks.connectionUpdate, - }, - $transaction: vi.fn(async (callback) => callback({ - connection: { - update: mocks.transactionConnectionUpdate, - }, - repo: { - upsert: mocks.transactionRepoUpsert, - }, - })), - }, -})); - vi.mock('./repoCompileUtils.js', () => ({ compileAzureDevOpsConfig: vi.fn(), compileBitbucketConfig: vi.fn(), @@ -67,7 +53,35 @@ vi.mock('./ee/syncSearchContexts.js', () => ({ syncSearchContexts: mocks.syncSearchContexts, })); -import { connectionWorkload } from './connectionWorkload.js'; +import { createConnectionWorkload } from './connectionWorkload.js'; + +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ + connection: { + update: mocks.transactionConnectionUpdate, + }, + repo: { + upsert: mocks.transactionRepoUpsert, + }, +})); + +const db = { + connection: { + findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, + update: mocks.connectionUpdate, + }, + connectionSyncJob: { + upsert: mocks.connectionSyncJobUpsert, + update: mocks.connectionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const connectionWorkload = createConnectionWorkload({ + db, + settings: { + maxConnectionSyncJobConcurrency: 2, + } as never, +}); const data = { connectionId: 42, @@ -79,6 +93,7 @@ const lifecycleContext = { jobId: 'job-1', attemptsMade: 0, maxAttempts: 2, + prisma: db, }; describe('connectionWorkload', () => { @@ -86,11 +101,65 @@ describe('connectionWorkload', () => { vi.clearAllMocks(); }); - test('does not declare database-backed lifecycle hooks', () => { - expect(connectionWorkload.queueSpec.onEnqueued).toBeUndefined(); - expect(connectionWorkload.onStarted).toBeUndefined(); - expect(connectionWorkload.onCompleted).toBeUndefined(); - expect(connectionWorkload.onTerminalFailure).toBeUndefined(); + test('declares database-backed lifecycle hooks', () => { + expect(connectionWorkload.onStarted).toBeTypeOf('function'); + expect(connectionWorkload.onCompleted).toBeTypeOf('function'); + expect(connectionWorkload.onTerminalFailure).toBeTypeOf('function'); + }); + + test('marks the connection sync job as in progress when started', async () => { + await connectionWorkload.onStarted?.(lifecycleContext); + + expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + update: { + status: 'IN_PROGRESS', + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: 'job-1', + connectionId: 42, + status: 'IN_PROGRESS', + warningMessages: [], + }, + }); + }); + + test('marks the connection sync job as completed', async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, undefined); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'COMPLETED', + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + }); + + test('marks the connection sync job as failed after terminal failure', async () => { + await connectionWorkload.onTerminalFailure?.( + lifecycleContext, + new Error('Connection credentials expired'), + ); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'FAILED', + completedAt: expect.any(Date), + errorMessage: 'Connection credentials expired', + }, + }); }); test('discovers repositories using the connection provider', async () => { @@ -136,6 +205,14 @@ describe('connectionWorkload', () => { }, ); expect(updateProgress).not.toHaveBeenCalled(); + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + warningMessages: ['Repository was archived'], + }, + }); expect(result).toBeUndefined(); }); }); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index 9c35951cc..a6f87353b 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -4,7 +4,7 @@ import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHost import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import * as Sentry from "@sentry/node"; -import { PrismaClient } from "@sourcebot/db"; +import { ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; interface Props { db: PrismaClient, @@ -14,7 +14,7 @@ interface Props { export const createConnectionWorkload = ({ db, settings -}: Props): Workload<'connection'> => ({ +}: Props): Workload<'connection-sync'> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, process: async ({ @@ -24,6 +24,7 @@ export const createConnectionWorkload = ({ }, logger, signal, + jobId, }) => { logger.info(`Syncing connection ${connectionId}`, { connectionId, @@ -43,7 +44,16 @@ export const createConnectionWorkload = ({ signal, }); - let { repoData } = result; + let { repoData, warnings } = result; + + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + warningMessages: warnings, + }, + }); logger.info(`Discovered ${repoData.length} repositories`, { connectionId, @@ -138,8 +148,51 @@ export const createConnectionWorkload = ({ logger.info(`Connection ${connectionId} sync finished`, { connectionId, }); - } -}) + }, + onStarted: async ({ data: { connectionId }, jobId }) => { + await db.connectionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: jobId, + connectionId, + status: ConnectionSyncJobStatus.IN_PROGRESS, + warningMessages: [], + }, + }); + }, + onCompleted: async ({ jobId }) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + }, + onTerminalFailure: async ({ jobId }, error) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + }, +}); const discoverConnectionRepositories = async ({ config, @@ -173,4 +226,4 @@ const discoverConnectionRepositories = async ({ return compileGenericGitHostConfig(config, connectionId); } } -}; \ No newline at end of file +}; diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 8e3fbce99..d45f24936 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -13,7 +13,7 @@ import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; -import { connectionWorkload, createConnectionWorkload } from "./connectionWorkload.js"; +import { createConnectionWorkload } from "./connectionWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 894efdade..2a6f1649f 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -111,10 +111,10 @@ describe('normalizeJobState', () => { }); const createWorkload = ( - overrides: Partial> = {}, -): Workload<'connection', { repoCount: number }> => ({ + overrides: Partial> = {}, +): Workload<'connection-sync', { repoCount: number }> => ({ queueSpec: { - name: 'connection', + name: 'connection-sync', dedupKey: ({ connectionId }) => `connection:${connectionId}`, jobOptions: { attempts: 2, @@ -150,7 +150,7 @@ describe('BullMQJobManager lifecycle', () => { const workload = createWorkload(); manager.register(workload); - const result = await manager.trigger('connection', data); + const result = await manager.trigger('connection-sync', data); expect(result).toBe('job-1'); expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); @@ -183,7 +183,7 @@ describe('BullMQJobManager lifecycle', () => { }); test('provides the structured job logger to the workload processor', async () => { - const process = vi.fn(async (context: ProcessContext<'connection'>) => { + const process = vi.fn(async (context: ProcessContext<'connection-sync'>) => { context.logger.info('Processing connection'); return { repoCount: 3 }; }); diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index 949af1c50..eadb37b88 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -53,14 +53,14 @@ describe('reconciliationWorkload', () => { vi.useRealTimers(); }); - test('runs every 15 minutes on the reconciliation queue', () => { + test('runs every 10 seconds on the reconciliation queue', () => { const workload = createReconciliationWorkload({ db, settings, }); expect(workload.queueSpec.name).toBe('reconciliation'); - expect(workload.schedule).toEqual({ every: '15m' }); + expect(workload.schedule).toEqual({ every: '10s' }); expect(workload.concurrency).toBe(1); }); @@ -100,11 +100,11 @@ describe('reconciliationWorkload', () => { }, }); expect(trigger).toHaveBeenCalledTimes(2); - expect(trigger).toHaveBeenCalledWith('connection', { + expect(trigger).toHaveBeenCalledWith('connection-sync', { connectionId: 42, orgId: 1, }); - expect(trigger).toHaveBeenCalledWith('connection', { + expect(trigger).toHaveBeenCalledWith('connection-sync', { connectionId: 84, orgId: 2, }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index f34008b2a..90a14f14e 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -31,7 +31,7 @@ export const createReconciliationWorkload = ({ await Promise.all(connections.map(async (connection) => { logger.debug(`Scheduling connection sync for connection ${connection.id}`); - await trigger('connection', { + await trigger('connection-sync', { connectionId: connection.id, orgId: connection.orgId, }); diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts new file mode 100644 index 000000000..076814b3d --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -0,0 +1,140 @@ +import type { PrismaClient } from '@sourcebot/db'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { createRepoIndexWorkload } from './repoIndexWorkload.js'; + +const repoIndexingJobUpsert = vi.fn(); +const repoIndexingJobUpdate = vi.fn(); +const repoUpdate = vi.fn(); +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ + repoIndexingJob: { + upsert: repoIndexingJobUpsert, + update: repoIndexingJobUpdate, + }, + repo: { + update: repoUpdate, + }, +})); + +const db = { + $transaction: transaction, +} as unknown as PrismaClient; + +const workload = createRepoIndexWorkload({ + db, + settings: { + maxRepoIndexingJobConcurrency: 2, + } as never, +}); + +const lifecycleContext = { + data: { + repoId: 42, + type: 'INDEX' as const, + }, + jobId: 'job-1', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, +}; + +describe('repoIndexWorkload lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test('declares database-backed lifecycle hooks', () => { + expect(workload.onStarted).toBeTypeOf('function'); + expect(workload.onCompleted).toBeTypeOf('function'); + expect(workload.onTerminalFailure).toBeTypeOf('function'); + }); + + test('marks the repo indexing job and repo as in progress when started', async () => { + await workload.onStarted?.(lifecycleContext); + + expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + update: { + status: 'IN_PROGRESS', + completedAt: null, + errorMessage: null, + }, + create: { + id: 'job-1', + repoId: 42, + type: 'INDEX', + status: 'IN_PROGRESS', + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: 'IN_PROGRESS', + }, + }); + }); + + test('marks the repo indexing job and repo as completed', async () => { + await workload.onCompleted?.(lifecycleContext, undefined); + + expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'COMPLETED', + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: 'COMPLETED', + }, + }); + }); + + test('does not update a completed cleanup job after its repo cascades the job row', async () => { + await workload.onCompleted?.({ + ...lifecycleContext, + data: { + repoId: 42, + type: 'CLEANUP', + }, + }, undefined); + + expect(transaction).not.toHaveBeenCalled(); + }); + + test('marks the repo indexing job and repo as failed after terminal failure', async () => { + await workload.onTerminalFailure?.( + lifecycleContext, + new Error('Unable to clone repository'), + ); + + expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + where: { + id: 'job-1', + }, + data: { + status: 'FAILED', + completedAt: expect.any(Date), + errorMessage: 'Unable to clone repository', + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: 'FAILED', + }, + }); + }); +}); diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index 3377d1c7e..7e21d9493 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -1,4 +1,4 @@ -import { PrismaClient, Repo } from "@sourcebot/db"; +import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { createLogger, getRepoPath, JobLogSink, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; import { existsSync } from 'fs'; import { readdir, rm } from 'fs/promises'; @@ -81,6 +81,84 @@ export const createRepoIndexWorkload = ({ } } }, + onStarted: async ({ data: { repoId, type }, jobId }) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoIndexingJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + type: RepoIndexingJobType[type], + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + }); + }, + onCompleted: async ({ data: { repoId, type }, jobId }) => { + // A successful cleanup deletes the Repo in `process`, which cascades to its + // RepoIndexingJob records. There is no row left to mark as completed. + if (type === RepoIndexingJobType.CLEANUP) { + return; + } + + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, + }, + }); + }); + }, + onTerminalFailure: async ({ data: { repoId }, jobId }, error) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, + }, + }); + }); + }, }); const indexRepository = async ( diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index c73b1fb5d..34955363c 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -2,7 +2,7 @@ import type { Redis } from 'ioredis'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import type { QueueSpec } from './queue.js'; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; -import { REPO_INDEX_QUEUE } from './queue.js'; +import { CONNECTION_QUEUE, REPO_INDEX_QUEUE } from './queue.js'; const queueMocks = vi.hoisted(() => ({ add: vi.fn(), @@ -32,8 +32,8 @@ vi.mock('bullmq', () => ({ import { BullMQClient } from './bullmqClient.js'; import { PrismaClient } from '@sourcebot/db'; -const connectionSpec: QueueSpec<'connection'> = { - name: 'connection', +const connectionSpec: QueueSpec<'connection-sync'> = { + name: 'connection-sync', dedupKey: ({ connectionId }) => `connection:${connectionId}`, jobOptions: { attempts: 2, @@ -61,7 +61,7 @@ describe('BullMQClient', () => { expect(result).toEqual(expect.any(String)); expect(queueMocks.add).toHaveBeenCalledWith( - 'connection', + 'connection-sync', data, expect.objectContaining({ jobId: result, @@ -97,11 +97,35 @@ describe('BullMQClient', () => { expect(onEnqueued).not.toHaveBeenCalled(); }); - test('stores the latest repo indexing job id when enqueueing a repo job', async () => { - const repoUpdate = vi.fn(); + test('upserts a pending connection sync job when enqueueing a connection sync', async () => { + const connectionSyncJobUpsert = vi.fn(); const client = new BullMQClient(redis, { - repo: { - update: repoUpdate, + connectionSyncJob: { + upsert: connectionSyncJobUpsert, + }, + } as unknown as PrismaClient); + + const result = await client.enqueue(CONNECTION_QUEUE, data); + + expect(connectionSyncJobUpsert).toHaveBeenCalledWith({ + where: { + id: result, + }, + update: {}, + create: { + id: result, + connectionId: 42, + status: 'PENDING', + warningMessages: [], + }, + }); + }); + + test('upserts a pending repo indexing job when enqueueing with repo-level deduplication', async () => { + const repoIndexingJobUpsert = vi.fn(); + const client = new BullMQClient(redis, { + repoIndexingJob: { + upsert: repoIndexingJobUpsert, }, } as unknown as PrismaClient); @@ -120,12 +144,16 @@ describe('BullMQClient', () => { deduplication: { id: 'repo:42' }, }), ); - expect(repoUpdate).toHaveBeenCalledWith({ + expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ where: { - id: 42, + id: result, }, - data: { - latestIndexingJobId: result, + update: {}, + create: { + id: result, + repoId: 42, + type: 'INDEX', + status: 'PENDING', }, }); }); diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 81e20c21f..7c65ac5ef 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,5 +1,5 @@ -import { PrismaClient } from "@sourcebot/db"; +import { ConnectionSyncJobStatus, PrismaClient, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; export type QueueName = keyof QueueRegistry; @@ -7,37 +7,50 @@ export type DataOf = QueueRegistry[TName]; type EmptyJobData = Record; interface QueueRegistry { - 'connection': { + 'reconciliation': EmptyJobData, + 'connection-sync': { connectionId: number, orgId: number }, - 'reconciliation': EmptyJobData, 'repo-index': { repoId: number, type: 'INDEX' | 'CLEANUP', }, } -export const CONNECTION_QUEUE: QueueSpec<'connection'> = { - name: 'connection', +export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { + name: 'reconciliation', jobOptions: { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, - dedupKey: (data) => `connection:${data.connectionId}`, -} - +}; -export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { - name: 'reconciliation', +export const CONNECTION_QUEUE: QueueSpec<'connection-sync'> = { + name: 'connection-sync', jobOptions: { attempts: 2, backoff: { type: 'exponential', delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, + dedupKey: (data) => `connection:${data.connectionId}`, + onEnqueued: async ({ prisma, data: { connectionId }, jobId }) => { + await prisma.connectionSyncJob.upsert({ + where: { + id: jobId, + }, + update: {}, + create: { + id: jobId, + connectionId, + status: ConnectionSyncJobStatus.PENDING, + warningMessages: [], + }, + }); + } }; export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { @@ -49,6 +62,20 @@ export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, dedupKey: (data) => `repo:${data.repoId}`, + onEnqueued: async ({ prisma, data: { repoId, type }, jobId }) => { + await prisma.repoIndexingJob.upsert({ + where: { + id: jobId, + }, + update: {}, + create: { + id: jobId, + repoId, + type: RepoIndexingJobType[type], + status: RepoIndexingJobStatus.PENDING, + }, + }); + }, }; export interface QueueSpec { From c7d9744b00bbc53dd21ead168c50fb49cac24557 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 3 Aug 2026 15:55:57 -0700 Subject: [PATCH 10/10] further wip --- docs/snippets/schemas/v3/index.schema.mdx | 20 +- packages/backend/src/bitbucket.ts | 2 +- .../backend/src/connectionWorkload.test.ts | 322 +++++++++-- packages/backend/src/connectionWorkload.ts | 157 +++-- .../ee/accountPermissionSyncWorkload.test.ts | 423 ++++++++++++++ .../src/ee/accountPermissionSyncWorkload.ts | 525 +++++++++++++++++ .../src/ee/accountPermissionSyncer.test.ts | 205 ------- .../backend/src/ee/accountPermissionSyncer.ts | 535 ------------------ .../src/ee/repoPermissionSyncWorkload.test.ts | 361 ++++++++++++ .../src/ee/repoPermissionSyncWorkload.ts | 407 +++++++++++++ .../backend/src/ee/repoPermissionSyncer.ts | 436 -------------- packages/backend/src/index.ts | 12 + packages/backend/src/jobManager.test.ts | 229 +++++--- packages/backend/src/jobManager.ts | 184 ++++-- .../src/reconciliationWorkload.test.ts | 199 +++++++ .../backend/src/reconciliationWorkload.ts | 200 +++++-- .../backend/src/repoIndexWorkload.test.ts | 99 ++-- packages/backend/src/types.ts | 60 +- packages/schemas/src/v3/index.schema.ts | 20 +- packages/schemas/src/v3/index.type.ts | 10 +- packages/shared/src/bullmqClient.test.ts | 251 -------- packages/shared/src/bullmqClient.ts | 22 - packages/shared/src/constants.ts | 8 +- packages/shared/src/env.server.ts | 1 - packages/shared/src/index.server.ts | 3 +- packages/shared/src/jobLogger.test.ts | 19 + packages/shared/src/jobLogger.ts | 33 +- packages/shared/src/queue.ts | 104 ++-- packages/web/src/lib/bullmqClient.ts | 3 +- schemas/v3/index.json | 10 +- 30 files changed, 2948 insertions(+), 1912 deletions(-) create mode 100644 packages/backend/src/ee/accountPermissionSyncWorkload.test.ts create mode 100644 packages/backend/src/ee/accountPermissionSyncWorkload.ts delete mode 100644 packages/backend/src/ee/accountPermissionSyncer.test.ts delete mode 100644 packages/backend/src/ee/accountPermissionSyncer.ts create mode 100644 packages/backend/src/ee/repoPermissionSyncWorkload.test.ts create mode 100644 packages/backend/src/ee/repoPermissionSyncWorkload.ts delete mode 100644 packages/backend/src/ee/repoPermissionSyncer.ts delete mode 100644 packages/shared/src/bullmqClient.test.ts diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 36e908e30..1cff30599 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -42,17 +42,17 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -96,12 +96,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, @@ -228,17 +228,17 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -282,12 +282,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index 9051164a7..84f85d023 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -755,7 +755,7 @@ export const getReposForAuthenticatedBitbucketServerUser = async ( * @note This only covers direct user-to-repo grants. It does NOT include users who have access via: * - Project-level permissions (inherited by all repos in the project) * - Group membership - * These users will still gain access through account-driven syncing (accountPermissionSyncer). + * These users will still gain access through account-driven permission syncing. * * @see https://developer.atlassian.com/server/bitbucket/rest/v906/api-group-repository/#api-rest-api-latest-projects-projectkey-repos-reposlug-permissions-users-get */ diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts index 679bd891b..d55deb752 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionWorkload.test.ts @@ -1,29 +1,31 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import type { PrismaClient } from '@sourcebot/db'; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { PrismaClient } from "@sourcebot/db"; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), connectionSyncJobUpsert: vi.fn(), connectionSyncJobUpdate: vi.fn(), - transactionConnectionUpdate: vi.fn(), - transactionRepoUpsert: vi.fn(), + repoFindMany: vi.fn(), + repoUpsert: vi.fn(), + repoToConnectionDeleteMany: vi.fn(), compileGithubConfig: vi.fn(), loadConfig: vi.fn(), syncSearchContexts: vi.fn(), })); -vi.mock('@sentry/node', () => ({ +vi.mock("@sentry/node", () => ({ captureException: vi.fn(), })); -vi.mock('@sourcebot/shared', () => ({ +vi.mock("@sourcebot/shared", () => ({ CONNECTION_QUEUE: { - name: 'connection-sync', - dedupKey: ({ connectionId }: { connectionId: number }) => `connection:${connectionId}`, + name: "connection-sync", + dedupKey: ({ connectionId }: { connectionId: number }) => + `connection:${connectionId}`, jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: 500, }, @@ -33,13 +35,13 @@ vi.mock('@sourcebot/shared', () => ({ error: vi.fn(), })), env: { - CONFIG_PATH: '/config.json', + CONFIG_PATH: "/config.json", CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: 60_000, }, loadConfig: mocks.loadConfig, })); -vi.mock('./repoCompileUtils.js', () => ({ +vi.mock("./repoCompileUtils.js", () => ({ compileAzureDevOpsConfig: vi.fn(), compileBitbucketConfig: vi.fn(), compileGenericGitHostConfig: vi.fn(), @@ -49,31 +51,28 @@ vi.mock('./repoCompileUtils.js', () => ({ compileGitlabConfig: vi.fn(), })); -vi.mock('./ee/syncSearchContexts.js', () => ({ +vi.mock("./ee/syncSearchContexts.js", () => ({ syncSearchContexts: mocks.syncSearchContexts, })); -import { createConnectionWorkload } from './connectionWorkload.js'; - -const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ - connection: { - update: mocks.transactionConnectionUpdate, - }, - repo: { - upsert: mocks.transactionRepoUpsert, - }, -})); +import { createConnectionWorkload } from "./connectionWorkload.js"; const db = { connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, }, + repo: { + findMany: mocks.repoFindMany, + upsert: mocks.repoUpsert, + }, + repoToConnection: { + deleteMany: mocks.repoToConnectionDeleteMany, + }, connectionSyncJob: { upsert: mocks.connectionSyncJobUpsert, update: mocks.connectionSyncJobUpdate, }, - $transaction: transaction, } as unknown as PrismaClient; const connectionWorkload = createConnectionWorkload({ @@ -88,94 +87,106 @@ const data = { orgId: 7, }; +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + const lifecycleContext = { data, - jobId: 'job-1', + jobId: "job-1", attemptsMade: 0, maxAttempts: 2, prisma: db, + logger: lifecycleLogger, }; -describe('connectionWorkload', () => { +describe("connectionWorkload", () => { beforeEach(() => { vi.clearAllMocks(); }); - test('declares database-backed lifecycle hooks', () => { - expect(connectionWorkload.onStarted).toBeTypeOf('function'); - expect(connectionWorkload.onCompleted).toBeTypeOf('function'); - expect(connectionWorkload.onTerminalFailure).toBeTypeOf('function'); + test("declares database-backed lifecycle hooks", () => { + expect(connectionWorkload.onStarted).toBeTypeOf("function"); + expect(connectionWorkload.onCompleted).toBeTypeOf("function"); + expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); }); - test('marks the connection sync job as in progress when started', async () => { + test("marks the connection sync job as in progress when started", async () => { await connectionWorkload.onStarted?.(lifecycleContext); expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, update: { - status: 'IN_PROGRESS', + status: "IN_PROGRESS", completedAt: null, errorMessage: null, warningMessages: [], }, create: { - id: 'job-1', + id: "job-1", connectionId: 42, - status: 'IN_PROGRESS', + status: "IN_PROGRESS", warningMessages: [], }, }); }); - test('marks the connection sync job as completed', async () => { - await connectionWorkload.onCompleted?.(lifecycleContext, undefined); + test("marks the connection sync job as completed", async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, { + reposToCleanup: [], + reposToIndex: [], + }); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'COMPLETED', + status: "COMPLETED", completedAt: expect.any(Date), errorMessage: null, }, }); }); - test('marks the connection sync job as failed after terminal failure', async () => { + test("marks the connection sync job as failed after terminal failure", async () => { await connectionWorkload.onTerminalFailure?.( lifecycleContext, - new Error('Connection credentials expired'), + new Error("Connection credentials expired"), ); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'FAILED', + status: "FAILED", completedAt: expect.any(Date), - errorMessage: 'Connection credentials expired', + errorMessage: "Connection credentials expired", }, }); }); - test('discovers repositories using the connection provider', async () => { + test("discovers repositories using the connection provider", async () => { const config = { - type: 'github' as const, + type: "github" as const, }; mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, - name: 'github', + name: "github", config, }); mocks.compileGithubConfig.mockResolvedValue({ repoData: [], - warnings: ['Repository was archived'], + warnings: ["Repository was archived"], }); mocks.connectionUpdate.mockResolvedValue({}); + mocks.repoFindMany.mockResolvedValue([]); mocks.loadConfig.mockResolvedValue({ contexts: undefined }); mocks.syncSearchContexts.mockResolvedValue(undefined); const updateProgress = vi.fn(); @@ -196,23 +207,220 @@ describe('connectionWorkload', () => { trigger: vi.fn(), }); - expect(mocks.compileGithubConfig).toHaveBeenCalledWith(config, 42, signal); - expect(logger.info).toHaveBeenCalledWith( - 'Discovered 0 repositories', - { - connectionId: 42, - repositoryCount: 0, - }, + expect(mocks.compileGithubConfig).toHaveBeenCalledWith( + config, + 42, + signal, ); + expect(logger.info).toHaveBeenCalledWith("Discovered 0 repositories", { + connectionId: 42, + repositoryCount: 0, + }); expect(updateProgress).not.toHaveBeenCalled(); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - warningMessages: ['Repository was archived'], + warningMessages: ["Repository was archived"], + }, + }); + expect(result).toEqual({ + reposToCleanup: [], + reposToIndex: [], + }); + }); + + test("finds orphaned repositories and repositories needing a first index", async () => { + const config = { + type: "github" as const, + }; + const existingIndexedAt = new Date("2026-07-30T12:00:00.000Z"); + const existingRepo = { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-1", + connections: { + create: { + connectionId: 42, + }, + }, + }; + const newRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-4", + connections: { + create: { + connectionId: 42, + }, + }, + }; + + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [existingRepo, newRepo], + warnings: [], + }); + mocks.repoFindMany + .mockResolvedValueOnce([{ id: 1 }, { id: 2 }, { id: 3 }]) + .mockResolvedValueOnce([{ id: 2, name: "github.com/sourcebot/repo-2" }]); + mocks.repoUpsert + .mockResolvedValueOnce({ + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt: existingIndexedAt, + }) + .mockResolvedValueOnce({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + mocks.repoToConnectionDeleteMany.mockResolvedValue({ count: 2 }); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + const trigger = vi.fn(); + + const result = await connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + }); + + expect(mocks.repoFindMany).toHaveBeenNthCalledWith(1, { + where: { + connections: { + some: { + connectionId: 42, + }, + }, }, + select: { + id: true, + }, + }); + expect(mocks.repoUpsert).toHaveBeenNthCalledWith(1, { + where: { + external_id_external_codeHostUrl_orgId: { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + orgId: 7, + }, + }, + update: { + ...existingRepo, + connections: { + createMany: { + data: { + connectionId: 42, + }, + skipDuplicates: true, + }, + }, + }, + create: existingRepo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }); + expect(mocks.repoToConnectionDeleteMany).toHaveBeenCalledWith({ + where: { + connectionId: 42, + repoId: { + in: [2, 3], + }, + }, + }); + expect(mocks.repoFindMany).toHaveBeenNthCalledWith(2, { + where: { + id: { + in: [2, 3], + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }); + expect(result).toEqual({ + reposToCleanup: [ + { id: 2, name: "github.com/sourcebot/repo-2" }, + ], + reposToIndex: [ + { id: 4, name: "github.com/sourcebot/repo-4" }, + ], + }); + expect(trigger).toHaveBeenNthCalledWith(1, "repo-index", { + repoId: 2, + type: "CLEANUP", + }); + expect(trigger).toHaveBeenNthCalledWith(2, "repo-index", { + repoId: 4, + type: "INDEX", + }); + expect(mocks.repoUpsert.mock.invocationCallOrder[1]).toBeLessThan( + mocks.repoToConnectionDeleteMany.mock.invocationCallOrder[0], + ); + expect(mocks.repoFindMany.mock.invocationCallOrder[1]).toBeLessThan( + trigger.mock.invocationCallOrder[0], + ); + }); + + test("does not mark the connection synced when scheduling fails", async () => { + const config = { + type: "github" as const, + }; + const newRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-4", + connections: { + create: { + connectionId: 42, + }, + }, + }; + + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [newRepo], + warnings: [], + }); + mocks.repoFindMany.mockResolvedValueOnce([]); + mocks.repoUpsert.mockResolvedValueOnce({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + const trigger = vi.fn().mockRejectedValue(new Error("Redis unavailable")); + + await expect(connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + })).rejects.toThrow("Redis unavailable"); + + expect(trigger).toHaveBeenCalledWith("repo-index", { + repoId: 4, + type: "INDEX", }); - expect(result).toBeUndefined(); + expect(mocks.connectionUpdate).not.toHaveBeenCalled(); + expect(mocks.syncSearchContexts).not.toHaveBeenCalled(); }); }); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts index a6f87353b..b6e429b50 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionWorkload.ts @@ -11,10 +11,15 @@ interface Props { settings: Settings; } +interface ConnectionSyncResult { + reposToCleanup: { id: number; name: string }[]; + reposToIndex: { id: number; name: string }[]; +} + export const createConnectionWorkload = ({ db, settings -}: Props): Workload<'connection-sync'> => ({ +}: Props): Workload<'connection-sync', ConnectionSyncResult> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, process: async ({ @@ -25,6 +30,7 @@ export const createConnectionWorkload = ({ logger, signal, jobId, + trigger, }) => { logger.info(`Syncing connection ${connectionId}`, { connectionId, @@ -68,59 +74,107 @@ export const createConnectionWorkload = ({ ); }) - // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, - // and then recreate them when we upsert the repos. For example, if a repo is no-longer - // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't - // appear in the repoData array above, and so the RepoToConnection record won't be re-created. - // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await db.$transaction(async (tx) => { - const deleteStart = performance.now(); - await tx.connection.update({ - where: { - id: connectionId, + const previouslyAssociatedRepos = await db.repo.findMany({ + where: { + connections: { + some: { + connectionId, + }, }, - data: { - repos: { - deleteMany: {} - } - } - }); - const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted existing repository associations`, { - connectionId, - connectionName: connection.name, - durationMs: deleteDuration, - }); + }, + select: { + id: true, + }, + }); + + const upsertedRepos: { id: number; name: string; indexedAt: Date | null }[] = []; - const totalUpsertStart = performance.now(); - for (const repo of repoData) { - const upsertStart = performance.now(); - await tx.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: repo.external_id, - external_codeHostUrl: repo.external_codeHostUrl, - orgId: orgId, - } + for (const repo of repoData) { + const upsertedRepo = await db.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, + orgId: orgId, + } + }, + update: { + ...repo, + connections: { + createMany: { + data: { + connectionId, + }, + skipDuplicates: true, + }, }, - update: repo, - create: repo, - }) - const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repository ${repo.displayName}`, { + }, + create: repo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }) + upsertedRepos.push(upsertedRepo); + } + + const currentRepoIds = new Set(upsertedRepos.map(({ id }) => id)); + const staleRepoIds = previouslyAssociatedRepos + .map(({ id }) => id) + .filter((id) => !currentRepoIds.has(id)); + + if (staleRepoIds.length > 0) { + await db.repoToConnection.deleteMany({ + where: { connectionId, - externalId: repo.external_id, - durationMs: upsertDuration, - }); - } - const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.info(`Stored ${repoData.length} repositories`, { - connectionId, - connectionName: connection.name, - repositoryCount: repoData.length, - durationMs: totalUpsertDuration, + repoId: { + in: staleRepoIds, + }, + }, }); - }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); + } + + const reposToCleanup = staleRepoIds.length > 0 + ? await db.repo.findMany({ + where: { + id: { + in: staleRepoIds, + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }) + : []; + + const reposToIndex = upsertedRepos + .filter(({ indexedAt }) => indexedAt === null) + .map(({ id, name }) => ({ id, name })); + + await Promise.all(reposToCleanup.map(({ id }) => + trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }) + )); + + await Promise.all(reposToIndex.map(({ id }) => + trigger('repo-index', { + repoId: id, + type: 'INDEX', + }) + )); + + logger.info(`Stored ${repoData.length} repositories`, { + connectionId, + connectionName: connection.name, + repositoryCount: repoData.length, + }); await db.connection.update({ where: { @@ -148,6 +202,11 @@ export const createConnectionWorkload = ({ logger.info(`Connection ${connectionId} sync finished`, { connectionId, }); + + return { + reposToCleanup, + reposToIndex, + }; }, onStarted: async ({ data: { connectionId }, jobId }) => { await db.connectionSyncJob.upsert({ diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..7326524a6 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -0,0 +1,423 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + ensureFreshAccountToken: vi.fn(), + getIdentityProviderConfig: vi.fn(), + getReposForAuthenticatedBitbucketCloudUser: vi.fn(), + getReposForAuthenticatedBitbucketServerUser: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + ACCOUNT_PERMISSION_SYNC_QUEUE: { + name: "account-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, + getIdentityProviderConfig: mocks.getIdentityProviderConfig, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser: + mocks.getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser: + mocks.getReposForAuthenticatedBitbucketServerUser, +})); + +vi.mock("./tokenRefresh.js", async (importOriginal) => ({ + ...(await importOriginal()), + ensureFreshAccountToken: mocks.ensureFreshAccountToken, +})); + +import { + classifyPermissionSyncFailure, + createAccountPermissionSyncWorkload, +} from "./accountPermissionSyncWorkload.js"; +import { + PermissionSyncUpstreamError, + type PermissionSyncUpstreamErrorKind, +} from "./permissionSyncError.js"; +import { + TokenRefreshError, + type TokenRefreshErrorKind, +} from "./tokenRefresh.js"; + +const tokenRefreshError = ( + kind: TokenRefreshErrorKind, + status?: number, +): TokenRefreshError => + new TokenRefreshError(`Token refresh failed: ${kind}`, { + kind, + status, + }); + +const upstreamError = ( + kind: PermissionSyncUpstreamErrorKind, +): PermissionSyncUpstreamError => + new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { + kind, + provider: "github", + operation: "list_accessible_repositories", + }); + +const account = { + id: "account_1", + providerId: "bitbucket-server", + issuerUrl: "https://bitbucket.example.com", + user: { email: "user@example.com" }, +}; +const accountFindUniqueOrThrow = vi.fn().mockResolvedValue(account); +const accountUpdate = vi.fn().mockResolvedValue(account); +const repoFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionDeleteMany = vi.fn().mockResolvedValue({ count: 95 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ account }); +const transaction = vi.fn((queries: Array>) => + Promise.all(queries), +); + +const db = { + account: { + findUniqueOrThrow: accountFindUniqueOrThrow, + update: accountUpdate, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + deleteMany: permissionDeleteMany, + }, + repo: { + findMany: repoFindMany, + }, + accountPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createAccountPermissionSyncWorkload({ + db, + settings: { + maxAccountPermissionSyncJobConcurrency: 2, + } as never, + }); + +const lifecycleContext = { + data: { + accountId: "account_1", + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.ensureFreshAccountToken.mockReset().mockResolvedValue("access-token"); + mocks.getIdentityProviderConfig.mockReset().mockResolvedValue({ + provider: "bitbucket-server", + baseUrl: "https://bitbucket.example.com", + }); + mocks.createBitbucketServerClient.mockReset().mockReturnValue({}); + mocks.getReposForAuthenticatedBitbucketServerUser + .mockReset() + .mockResolvedValue([]); + accountFindUniqueOrThrow.mockResolvedValue(account); + repoFindMany.mockResolvedValue([]); + permissionCreateMany.mockResolvedValue({ count: 0 }); + permissionDeleteMany.mockResolvedValue({ count: 95 }); + permissionSyncJobUpdate.mockResolvedValue({ account }); +}); + +describe("classifyPermissionSyncFailure", () => { + test("fails closed when the refresh token is rejected", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("refresh_token_rejected", 400), + ), + ).toEqual({ + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + }); + }); + + test.each([ + ["transient", 500], + ["configuration", 400], + ["invalid_response", undefined], + ["local_credential", undefined], + ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)( + "keeps permissions for a %s token refresh failure", + (kind, status) => { + expect( + classifyPermissionSyncFailure(tokenRefreshError(kind, status)), + ).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test("does not treat a token refresh configuration error with HTTP 401 as an API authorization failure", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("configuration", 401), + ), + ).toEqual({ + action: "preserve_permissions", + }); + }); + + test.each([ + ["credential_rejected", "upstream_credential_rejected"], + ["insufficient_scope", "upstream_insufficient_scope"], + ] as const)( + "fails closed for a classified %s upstream failure", + (kind, reason) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "clear_permissions", + reason, + }); + }, + ); + + test.each([ + "rate_limited", + "upstream_unavailable", + "forbidden", + "unknown", + ] satisfies PermissionSyncUpstreamErrorKind[])( + "keeps permissions for a classified %s upstream failure", + (kind) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test.each([401, 403, 410])( + "does not fail closed on an unclassified HTTP %s error", + (status) => { + const error = Object.assign(new Error(`HTTP ${status}`), { + status, + }); + expect(classifyPermissionSyncFailure(error)).toEqual({ + action: "preserve_permissions", + }); + }, + ); +}); + +describe("accountPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("account-permission-sync"); + expect(workload.concurrency).toBe(2); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("syncs the requested account", async () => { + const workload = createWorkload(); + + await workload.process(processContext); + + expect(accountFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: "account_1" }, + include: { user: true }, + }); + expect(mocks.ensureFreshAccountToken).toHaveBeenCalledWith(account, db); + expect(mocks.createBitbucketServerClient).toHaveBeenCalledWith( + "https://bitbucket.example.com", + undefined, + "access-token", + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(accountFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("atomically records a reauthentication issue when the refresh token is rejected", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).toHaveBeenCalledWith({ + where: { accountId: "account_1" }, + }); + expect(accountUpdate).toHaveBeenCalledWith({ + where: { id: "account_1" }, + data: { + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + permissionSyncIssueAt: expect.any(Date), + }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an issue even when permissions were cleared by an earlier attempt", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + permissionDeleteMany.mockResolvedValue({ count: 0 }); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + }), + }), + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an insufficient-scope issue for scope failures", async () => { + const error = upstreamError("insufficient_scope"); + mocks.getReposForAuthenticatedBitbucketServerUser.mockRejectedValue( + error, + ); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "INSUFFICIENT_SCOPE", + }), + }), + ); + }); + + test("preserves permissions for a transient refresh failure", async () => { + const error = tokenRefreshError("transient", 500); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).not.toHaveBeenCalled(); + expect(accountUpdate).not.toHaveBeenCalled(); + expect(transaction).not.toHaveBeenCalled(); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + accountId: "account_1", + status: "IN_PROGRESS", + }, + }); + }); + + test("marks a job completed and clears the account issue", async () => { + await createWorkload().onCompleted?.(lifecycleContext, undefined); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + account: { + update: { + permissionSyncedAt: expect.any(Date), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }, + }, + select: { + account: { + include: { user: true }, + }, + }, + }); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + }), + ); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "account-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.ts new file mode 100644 index 000000000..af5a3b721 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.ts @@ -0,0 +1,525 @@ +import * as Sentry from "@sentry/node"; +import { + Account, + AccountPermissionSyncIssue, + AccountPermissionSyncJobStatus, + PermissionSyncSource, + PrismaClient, +} from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + getIdentityProviderConfig, + JobLogSink, +} from "@sourcebot/shared"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser, +} from "../bitbucket.js"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, + getReposForAuthenticatedUser, +} from "../github.js"; +import { + createGitLabFromOAuthToken, + getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, + getProjectsForAuthenticatedUser, +} from "../gitlab.js"; +import { + PermissionSyncUpstreamError, + withPermissionSyncUpstreamError, +} from "./permissionSyncError.js"; +import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; +import { Settings, Workload } from "../types.js"; +import { IdentityProviderConfig } from "@sourcebot/schemas/v3/index.type"; + +type AccountWithUser = Account & { user: { email: string | null } }; + +type SupportedProvider = + | "github" + | "gitlab" + | "bitbucket-cloud" + | "bitbucket-server"; +type ProviderConfig = Extract< + IdentityProviderConfig, + { provider: TProvider } +>; + +interface ProviderPermissionSyncProps { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: ProviderConfig; +} + +export type PermissionCleanupReason = + | "oauth_refresh_token_rejected" + | "upstream_credential_rejected" + | "upstream_insufficient_scope"; + +export type PermissionCleanupDecision = + | { + action: "clear_permissions"; + reason: PermissionCleanupReason; + } + | { + action: "preserve_permissions"; + }; + +export const classifyPermissionSyncFailure = ( + error: unknown, +): PermissionCleanupDecision => { + // Token refresh failures have their own classification. Do not fall through + // to the generic HTTP checks because another token endpoint failure may + // also carry a 401 or 403 status. + if (error instanceof TokenRefreshError) { + return error.kind === "refresh_token_rejected" + ? { + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + } + : { action: "preserve_permissions" }; + } + + if (error instanceof PermissionSyncUpstreamError) { + if (error.kind === "credential_rejected") { + return { + action: "clear_permissions", + reason: "upstream_credential_rejected", + }; + } + if (error.kind === "insufficient_scope") { + return { + action: "clear_permissions", + reason: "upstream_insufficient_scope", + }; + } + } + + return { action: "preserve_permissions" }; +}; + +const PERMISSION_CLEANUP_DETAILS: Record< + PermissionCleanupReason, + { + message: string; + issue: AccountPermissionSyncIssue; + } +> = { + oauth_refresh_token_rejected: { + message: "OAuth refresh token rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_credential_rejected: { + message: "upstream credential rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_insufficient_scope: { + message: "insufficient OAuth scope", + issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, + }, +}; + +interface AccountPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createAccountPermissionSyncWorkload = ({ + db, + settings, +}: AccountPermissionSyncWorkloadDependencies): Workload<"account-permission-sync"> => { + return { + queueSpec: ACCOUNT_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxAccountPermissionSyncJobConcurrency, + process: async ({ data: { accountId }, logger: jobLogger }) => { + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + const account = await db.account.findUniqueOrThrow({ + where: { + id: accountId, + }, + include: { + user: true, + }, + }); + + jobLogger.debug( + `Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`, + ); + + try { + // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. + const accessToken = await ensureFreshAccountToken(account, db); + + const idpConfig = await getIdentityProviderConfig( + account.providerId, + ); + if (!idpConfig) { + throw new Error( + "Unable to find IDP config in config.json.", + ); + } + + const repoIds = await getAccessibleRepoIds({ + db, + account, + accessToken, + config: idpConfig, + }); + + await db.$transaction([ + db.account.update({ + where: { + id: account.id, + }, + data: { + accessibleRepos: { + deleteMany: {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: repoIds.map((repoId) => ({ + accountId: account.id, + repoId, + source: PermissionSyncSource.ACCOUNT_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + } catch (error) { + // Clear cached permissions only for classified permanent failures. + // Ambiguous HTTP errors and transient upstream failures preserve the + // last successful permission state. + const cleanupDecision = classifyPermissionSyncFailure(error); + + if (cleanupDecision.action === "clear_permissions") { + const details = + PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; + const [{ count }] = await db.$transaction([ + db.accountToRepoPermission.deleteMany({ + where: { accountId: account.id }, + }), + db.account.update({ + where: { id: account.id }, + data: { + permissionSyncIssue: details.issue, + permissionSyncIssueAt: new Date(), + }, + }), + ]); + const message = + error instanceof Error ? error.message : String(error); + jobLogger.warn( + `Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? "unknown"}) — fail-closed cleanup triggered by ${details.message}: ${message}`, + ); + } + throw error; + } + }, + onStarted: async ({ data: { accountId }, jobId }) => { + await db.accountPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + accountId, + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId, logger: jobLogger }) => { + const { account } = await db.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + account: { + update: { + permissionSyncedAt: new Date(), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + + jobLogger.debug( + `Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`, + ); + }, + onTerminalFailure: async ( + { data: { accountId }, jobId, logger: jobLogger }, + error, + ) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: ACCOUNT_PERMISSION_SYNC_QUEUE.name, + }, + }); + + const { account } = await db.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + + jobLogger.error( + `Account permission sync job failed for account (id: ${accountId}) for user ${account.user.email ?? "unknown user (email not found)"}: ${error.message}`, + ); + }, + }; +}; + +const getAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: IdentityProviderConfig; +}): Promise => { + switch (config.provider) { + case "github": + return getGitHubAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "gitlab": + return getGitLabAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-cloud": + return getBitbucketCloudAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-server": + return getBitbucketServerAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + default: + throw new Error(`Unsupported provider type: ${config.provider}`); + } +}; + +const getGitHubAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"github">): Promise => { + const { octokit } = await createOctokitFromToken({ + token: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "github", + "inspect_token_scopes", + () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), + ); + + // Token supports scope introspection (classic PAT or OAuth app token). + if (scopes !== null && !scopes.includes("repo")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, + { + kind: "insufficient_scope", + provider: "github", + operation: "inspect_token_scopes", + }, + ); + } + + // Public repos do not need an explicit permission mapping. + const githubRepos = await withPermissionSyncUpstreamError( + "github", + "list_accessible_repositories", + () => getReposForAuthenticatedUser("private", octokit), + ); + const gitHubRepoIds = githubRepos.map((repo) => repo.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "github", + external_id: { + in: gitHubRepoIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getGitLabAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"gitlab">): Promise => { + const api = await createGitLabFromOAuthToken({ + oauthToken: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "gitlab", + "inspect_token_scopes", + () => getGitLabOAuthScopesForAuthenticatedUser(api), + ); + if (!scopes.includes("read_api")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'read_api' scope required for permission syncing.`, + { + kind: "insufficient_scope", + provider: "gitlab", + operation: "inspect_token_scopes", + }, + ); + } + + // Public and internal repos do not need an explicit permission mapping. + const gitLabProjectIds = ( + await withPermissionSyncUpstreamError( + "gitlab", + "list_accessible_repositories", + () => getProjectsForAuthenticatedUser("private", api), + ) + ).map((project) => project.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "gitlab", + external_id: { + in: gitLabProjectIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketCloudAccessibleRepoIds = async ({ + db, + account, + accessToken, +}: ProviderPermissionSyncProps<"bitbucket-cloud">): Promise => { + // Use a bearer token by omitting the user. + const client = createBitbucketCloudClient(undefined, accessToken); + const bitbucketRepos = await withPermissionSyncUpstreamError( + "bitbucket-cloud", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketCloudUser(client), + ); + const bitbucketRepoUuids = bitbucketRepos.map((repo) => repo.uuid); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketCloud", + external_id: { + in: bitbucketRepoUuids, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketServerAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"bitbucket-server">): Promise => { + const client = createBitbucketServerClient( + config.baseUrl, + undefined, + accessToken, + ); + const serverRepos = await withPermissionSyncUpstreamError( + "bitbucket-server", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketServerUser(client), + ); + const serverRepoIds = serverRepos.map((repo) => repo.id); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketServer", + external_id: { in: serverRepoIds }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; diff --git a/packages/backend/src/ee/accountPermissionSyncer.test.ts b/packages/backend/src/ee/accountPermissionSyncer.test.ts deleted file mode 100644 index 09b35d594..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.test.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -const mocks = vi.hoisted(() => ({ - hasEntitlement: vi.fn(), -})); - -vi.mock('../entitlements.js', () => ({ - hasEntitlement: mocks.hasEntitlement, -})); - -import { AccountPermissionSyncer, classifyPermissionSyncFailure } from './accountPermissionSyncer.js'; -import { - PermissionSyncUpstreamError, - type PermissionSyncUpstreamErrorKind, -} from './permissionSyncError.js'; -import { TokenRefreshError, type TokenRefreshErrorKind } from './tokenRefresh.js'; - -const tokenRefreshError = ( - kind: TokenRefreshErrorKind, - status?: number, -): TokenRefreshError => new TokenRefreshError(`Token refresh failed: ${kind}`, { - kind, - status, -}); - -const upstreamError = ( - kind: PermissionSyncUpstreamErrorKind, -): PermissionSyncUpstreamError => new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { - kind, - provider: 'github', - operation: 'list_accessible_repositories', -}); - -const createSyncerHarness = (syncError?: Error, permissionCount = 95) => { - const account = { - id: 'account_1', - providerId: 'bitbucket-server', - user: { email: 'user@example.com' }, - }; - const db = { - accountPermissionSyncJob: { - update: vi.fn().mockResolvedValue({ account }), - }, - accountToRepoPermission: { - deleteMany: vi.fn().mockResolvedValue({ count: permissionCount }), - }, - account: { - update: vi.fn().mockResolvedValue(account), - }, - $transaction: vi.fn((queries: Array>) => Promise.all(queries)), - }; - const syncAccountPermissions = syncError - ? vi.fn().mockRejectedValue(syncError) - : vi.fn().mockResolvedValue(undefined); - const syncer = Object.create(AccountPermissionSyncer.prototype) as { - db: typeof db; - syncAccountPermissions: typeof syncAccountPermissions; - runJob(job: { data: { jobId: string } }): Promise; - onJobCompleted(job: { data: { jobId: string } }): Promise; - }; - syncer.db = db; - syncer.syncAccountPermissions = syncAccountPermissions; - - return { - account, - db, - job: { data: { jobId: 'job_1' } }, - syncer, - }; -}; - -beforeEach(() => { - vi.clearAllMocks(); - mocks.hasEntitlement.mockResolvedValue(true); -}); - -describe('classifyPermissionSyncFailure', () => { - test('fails closed when the refresh token is rejected', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('refresh_token_rejected', 400))).toEqual({ - action: 'clear_permissions', - reason: 'oauth_refresh_token_rejected', - }); - }); - - test.each([ - ['transient', 500], - ['configuration', 400], - ['invalid_response', undefined], - ['local_credential', undefined], - ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)('keeps permissions for a %s token refresh failure', (kind, status) => { - expect(classifyPermissionSyncFailure(tokenRefreshError(kind, status))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test('does not treat a token refresh configuration error with HTTP 401 as an API authorization failure', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('configuration', 401))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([ - ['credential_rejected', 'upstream_credential_rejected'], - ['insufficient_scope', 'upstream_insufficient_scope'], - ] as const)('fails closed for a classified %s upstream failure', (kind, reason) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'clear_permissions', - reason, - }); - }); - - test.each([ - 'rate_limited', - 'upstream_unavailable', - 'forbidden', - 'unknown', - ] satisfies PermissionSyncUpstreamErrorKind[])('keeps permissions for a classified %s upstream failure', (kind) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([401, 403, 410])('does not fail closed on an unclassified HTTP %s error', (status) => { - const error = Object.assign(new Error(`HTTP ${status}`), { status }); - expect(classifyPermissionSyncFailure(error)).toEqual({ - action: 'preserve_permissions', - }); - }); -}); - -describe('permission sync issue lifecycle', () => { - test('atomically records a reauthentication issue when the refresh token is rejected', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).toHaveBeenCalledWith({ - where: { accountId: 'account_1' }, - }); - expect(db.account.update).toHaveBeenCalledWith({ - where: { id: 'account_1' }, - data: { - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - permissionSyncIssueAt: expect.any(Date), - }, - }); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an issue when permissions were already cleared by an earlier attempt', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error, 0); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - }), - })); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an insufficient-scope issue for scope failures', async () => { - const error = upstreamError('insufficient_scope'); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'INSUFFICIENT_SCOPE', - }), - })); - }); - - test('does not record an issue or clear permissions for a transient refresh failure', async () => { - const error = tokenRefreshError('transient', 500); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).not.toHaveBeenCalled(); - expect(db.account.update).not.toHaveBeenCalled(); - expect(db.$transaction).not.toHaveBeenCalled(); - }); - - test('clears the action-required issue after a successful permission sync', async () => { - const { db, job, syncer } = createSyncerHarness(); - - await syncer.onJobCompleted(job); - - expect(db.accountPermissionSyncJob.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - account: { - update: expect.objectContaining({ - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }), - }, - }), - })); - }); -}); diff --git a/packages/backend/src/ee/accountPermissionSyncer.ts b/packages/backend/src/ee/accountPermissionSyncer.ts deleted file mode 100644 index 7184a191e..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.ts +++ /dev/null @@ -1,535 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { - PrismaClient, - AccountPermissionSyncIssue, - AccountPermissionSyncJobStatus, - Account, - PermissionSyncSource, -} from "@sourcebot/db"; -import { env, createLogger, getIdentityProviderConfig, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { Redis } from "ioredis"; -import { - createOctokitFromToken, - getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, - getReposForAuthenticatedUser, -} from "../github.js"; -import { - createGitLabFromOAuthToken, - getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, - getProjectsForAuthenticatedUser, -} from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getReposForAuthenticatedBitbucketCloudUser, getReposForAuthenticatedBitbucketServerUser } from "../bitbucket.js"; -import { Settings } from "../types.js"; -import { setIntervalAsync } from "../utils.js"; -import { PermissionSyncUpstreamError, withPermissionSyncUpstreamError } from "./permissionSyncError.js"; - -const LOG_TAG = 'user-permission-syncer'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -const QUEUE_NAME = 'accountPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; - -type AccountPermissionSyncJob = { - jobId: string; -} - -export type PermissionCleanupReason = - | 'oauth_refresh_token_rejected' - | 'upstream_credential_rejected' - | 'upstream_insufficient_scope'; - -export type PermissionCleanupDecision = - | { - action: 'clear_permissions'; - reason: PermissionCleanupReason; - } - | { - action: 'preserve_permissions'; - }; - -const PERMISSION_CLEANUP_DETAILS: Record = { - oauth_refresh_token_rejected: { - message: 'OAuth refresh token rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_credential_rejected: { - message: 'upstream credential rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_insufficient_scope: { - message: 'insufficient OAuth scope', - issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, - }, -}; - -export const classifyPermissionSyncFailure = (error: unknown): PermissionCleanupDecision => { - // Token refresh failures have their own classification. Do not fall through - // to the generic HTTP checks because another token endpoint failure may - // also carry a 401 or 403 status. - if (error instanceof TokenRefreshError) { - return error.kind === 'refresh_token_rejected' - ? { action: 'clear_permissions', reason: 'oauth_refresh_token_rejected' } - : { action: 'preserve_permissions' }; - } - - if (error instanceof PermissionSyncUpstreamError) { - if (error.kind === 'credential_rejected') { - return { action: 'clear_permissions', reason: 'upstream_credential_rejected' }; - } - if (error.kind === 'insufficient_scope') { - return { action: 'clear_permissions', reason: 'upstream_insufficient_scope' }; - } - } - - return { action: 'preserve_permissions' }; -}; - -export class AccountPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxAccountPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - const thresholdDate = new Date(Date.now() - this.settings.userDrivenPermissionSyncIntervalMs); - - const accounts = await this.db.account.findMany({ - where: { - AND: [ - { - providerType: { - in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS - } - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - AccountPermissionSyncJobStatus.PENDING, - AccountPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: AccountPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(accounts); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - public async schedulePermissionSyncForAccount(account: Account) { - const [job] = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: [{ accountId: account.id }], - }); - - await this.queue.add('accountPermissionSyncJob', { - jobId: job.id, - }, { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - priority: 1, - }); - - return job.id; - } - - private async schedulePermissionSync(accounts: Account[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: accounts.map(account => ({ - accountId: account.id, - })), - include: { - account: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'accountPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.account.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - try { - await this.syncAccountPermissions(account, logger); - } catch (error) { - // Clear cached permissions only for classified permanent failures. - // Ambiguous HTTP errors and transient upstream failures preserve the - // last successful permission state. - const cleanupDecision = classifyPermissionSyncFailure(error); - - if (cleanupDecision.action === 'clear_permissions') { - const details = PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; - const [{ count }] = await this.db.$transaction([ - this.db.accountToRepoPermission.deleteMany({ - where: { accountId: account.id }, - }), - this.db.account.update({ - where: { id: account.id }, - data: { - permissionSyncIssue: details.issue, - permissionSyncIssueAt: new Date(), - }, - }), - ]); - const message = error instanceof Error ? error.message : String(error); - logger.warn(`Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? 'unknown'}) — fail-closed cleanup triggered by ${details.message}: ${message}`); - } - throw error; - } - } - - private async syncAccountPermissions( - account: Account & { user: { email: string | null } }, - logger: ReturnType, - ) { - logger.debug(`Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`); - - // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. - const accessToken = await ensureFreshAccountToken(account, this.db); - - // Get a list of all repos that the user has access to from all connected accounts. - const repoIds = await (async () => { - const aggregatedRepoIds: Set = new Set(); - - const idpConfig = await getIdentityProviderConfig(account.providerId); - - if (!idpConfig) { - throw new Error(`Unable to find IDP config in config.json.`); - } - - if (idpConfig.provider === 'github') { - const { octokit } = await createOctokitFromToken({ - token: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'github', - 'inspect_token_scopes', - () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), - ); - - // Token supports scope introspection (classic PAT or OAuth app token) - if (scopes !== null) { - if (!scopes.includes('repo')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, - { - kind: 'insufficient_scope', - provider: 'github', - operation: 'inspect_token_scopes', - }, - ); - } - } - - // @note: we only care about the private repos since we don't need to build a mapping - // for public repos. - // @see: packages/web/src/prisma.ts - const githubRepos = await withPermissionSyncUpstreamError( - 'github', - 'list_accessible_repositories', - () => getReposForAuthenticatedUser(/* visibility = */ 'private', octokit), - ); - const gitHubRepoIds = githubRepos.map(repo => repo.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'github', - external_id: { - in: gitHubRepoIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'gitlab') { - const api = await createGitLabFromOAuthToken({ - oauthToken: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'gitlab', - 'inspect_token_scopes', - () => getGitLabOAuthScopesForAuthenticatedUser(api), - ); - if (!scopes.includes('read_api')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'read_api' scope required for permission syncing.`, - { - kind: 'insufficient_scope', - provider: 'gitlab', - operation: 'inspect_token_scopes', - }, - ); - } - - // @note: we only care about the private repos since we don't need to build a - // mapping for public or internal repos. Note that internal repos are _not_ - // enforced by permission syncing and therefore we don't need to fetch them - // here. - // - // @see: packages/web/src/prisma.ts - const gitLabProjectIds = ( - await withPermissionSyncUpstreamError( - 'gitlab', - 'list_accessible_repositories', - () => getProjectsForAuthenticatedUser('private', api), - ) - ).map(project => project.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'gitlab', - external_id: { - in: gitLabProjectIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-cloud') { - // @note: we don't pass a user here since we want to use a bearer token - // for authentication. - const client = createBitbucketCloudClient(/* user = */ undefined, accessToken) - const bitbucketRepos = await withPermissionSyncUpstreamError( - 'bitbucket-cloud', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketCloudUser(client), - ); - const bitbucketRepoUuids = bitbucketRepos.map(repo => repo.uuid); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketCloud', - external_id: { - in: bitbucketRepoUuids, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-server') { - const client = createBitbucketServerClient(idpConfig.baseUrl, /* user = */ undefined, accessToken); - const serverRepos = await withPermissionSyncUpstreamError( - 'bitbucket-server', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketServerUser(client), - ); - const serverRepoIds = serverRepos.map(r => r.id); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketServer', - external_id: { in: serverRepoIds }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else { - throw new Error(`Unsupported provider type: ${idpConfig.provider}`); - } - - return Array.from(aggregatedRepoIds); - })(); - - await this.db.$transaction([ - this.db.account.update({ - where: { - id: account.id, - }, - data: { - accessibleRepos: { - deleteMany: {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: repoIds.map(repoId => ({ - accountId: account.id, - repoId, - source: PermissionSyncSource.ACCOUNT_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.COMPLETED, - account: { - update: { - permissionSyncedAt: new Date(), - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }, - }, - completedAt: new Date(), - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.debug(`Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (accountId: string, email: string) => `Account permission sync job failed for account (id: ${accountId}) for user ${email}: ${err.message}`; - - if (job) { - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.error(errorMessage(account.id, account.user.email ?? 'unknown user (email not found)')); - } else { - logger.error(errorMessage('unknown account (id not found)', 'unknown user (id not found)')); - } - } -} diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..648c5b88d --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -0,0 +1,361 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + createGitLabFromPersonalAccessToken: vi.fn(), + createOctokitFromToken: vi.fn(), + getAuthCredentialsForRepo: vi.fn(), + getExplicitUserPermissionsForCloudRepo: vi.fn(), + getProjectMembers: vi.fn(), + getRepoCollaborators: vi.fn(), + getUserPermissionsForServerRepo: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + REPO_PERMISSION_SYNC_QUEUE: { + name: "repo-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../utils.js", () => ({ + getAuthCredentialsForRepo: mocks.getAuthCredentialsForRepo, +})); + +vi.mock("../github.js", () => ({ + createOctokitFromToken: mocks.createOctokitFromToken, + getRepoCollaborators: mocks.getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME: "github.com", +})); + +vi.mock("../gitlab.js", () => ({ + createGitLabFromPersonalAccessToken: + mocks.createGitLabFromPersonalAccessToken, + getProjectMembers: mocks.getProjectMembers, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo: + mocks.getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo: mocks.getUserPermissionsForServerRepo, +})); + +import { createRepoPermissionSyncWorkload } from "./repoPermissionSyncWorkload.js"; + +const repo = { + id: 42, + name: "github.com/sourcebot-dev/sourcebot", + displayName: "sourcebot-dev/sourcebot", + external_codeHostType: "github", + external_id: "123", + metadata: {}, + connections: [], +}; +const repoFindUniqueOrThrow = vi.fn().mockResolvedValue(repo); +const repoUpdate = vi.fn().mockResolvedValue(repo); +const accountFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ repo }); +const transaction = vi.fn((queries: Array>) => + Promise.all(queries), +); + +const db = { + repo: { + findUniqueOrThrow: repoFindUniqueOrThrow, + update: repoUpdate, + }, + account: { + findMany: accountFindMany, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + }, + repoPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createRepoPermissionSyncWorkload({ + db, + settings: { + maxRepoPermissionSyncJobConcurrency: 3, + } as never, + }); + +const lifecycleContext = { + data: { + repoId: 42, + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.getAuthCredentialsForRepo.mockReset().mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + mocks.createOctokitFromToken.mockReset().mockResolvedValue({ octokit: {} }); + mocks.getRepoCollaborators.mockReset().mockResolvedValue([]); + repoFindUniqueOrThrow.mockResolvedValue(repo); + permissionSyncJobUpdate.mockResolvedValue({ repo }); + accountFindMany.mockResolvedValue([]); + repoUpdate.mockResolvedValue(repo); + permissionCreateMany.mockResolvedValue({ count: 0 }); +}); + +describe("repoPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("repo-permission-sync"); + expect(workload.concurrency).toBe(3); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("syncs the requested repo with its connections", async () => { + await createWorkload().process(processContext); + + expect(repoFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + expect(mocks.getAuthCredentialsForRepo).toHaveBeenCalledWith( + repo, + jobLogger, + ); + expect(mocks.createOctokitFromToken).toHaveBeenCalledWith({ + token: "token", + url: undefined, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + + await expect(createWorkload().process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(repoFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("replaces all permissions for a complete GitHub sync", async () => { + const githubRepo = { + ...repo, + external_codeHostType: "github", + external_id: "123", + metadata: {}, + }; + repoFindUniqueOrThrow.mockResolvedValue(githubRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + const octokit = {}; + mocks.createOctokitFromToken.mockResolvedValue({ octokit }); + mocks.getRepoCollaborators.mockResolvedValue([{ id: 101 }]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(accountFindMany).toHaveBeenCalledWith({ + where: { + providerType: "github", + providerAccountId: { + in: ["101"], + }, + issuerUrl: "https://github.com", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: {}, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("preserves account-driven permissions for a partial Bitbucket Cloud sync", async () => { + const bitbucketRepo = { + ...repo, + external_codeHostType: "bitbucketCloud", + external_id: "repo-uuid", + metadata: { + codeHostMetadata: { + bitbucketCloud: { + workspace: "sourcebot", + repoSlug: "sourcebot", + }, + }, + }, + }; + repoFindUniqueOrThrow.mockResolvedValue(bitbucketRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://bitbucket.org", + token: "token", + connectionConfig: { + user: "service-account", + }, + }); + mocks.createBitbucketCloudClient.mockReturnValue({}); + mocks.getExplicitUserPermissionsForCloudRepo.mockResolvedValue([ + { accountId: "upstream-account" }, + ]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: { + source: "REPO_DRIVEN", + }, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + repoId: 42, + status: "IN_PROGRESS", + }, + }); + }); + + test("marks a job completed and updates the repo sync timestamp", async () => { + await createWorkload().onCompleted?.(lifecycleContext, undefined); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + repo: { + update: { + permissionSyncedAt: expect.any(Date), + }, + }, + }, + select: { + repo: true, + }, + }); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + select: { + repo: true, + }, + }); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "repo-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts new file mode 100644 index 000000000..7fd1e59df --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -0,0 +1,407 @@ +import * as Sentry from "@sentry/node"; +import { + PermissionSyncSource, + PrismaClient, + RepoPermissionSyncJobStatus, +} from "@sourcebot/db"; +import { + JobLogSink, + REPO_PERMISSION_SYNC_QUEUE, + repoMetadataSchema, +} from "@sourcebot/shared"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME, +} from "../github.js"; +import { + createGitLabFromPersonalAccessToken, + getProjectMembers, +} from "../gitlab.js"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo, +} from "../bitbucket.js"; +import { + RepoAuthCredentials, + RepoWithConnections, + Settings, + Workload, +} from "../types.js"; +import { getAuthCredentialsForRepo } from "../utils.js"; +import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; + +interface RepoPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createRepoPermissionSyncWorkload = ({ + db, + settings, +}: RepoPermissionSyncWorkloadDependencies): Workload<"repo-permission-sync"> => ({ + queueSpec: REPO_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxRepoPermissionSyncJobConcurrency, + process: async ({ data: { repoId }, logger }) => { + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + const repo = await db.repo.findUniqueOrThrow({ + where: { + id: repoId, + }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + const id = repo.id; + logger.debug(`Syncing permissions for repo ${repo.displayName}...`); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + if (!credentials) { + throw new Error(`No credentials found for repo ${id}`); + } + + const { accountIds, isPartialSync = false } = + await getPermissionSyncResult({ + db, + repo, + credentials, + logger: logger, + }); + + await db.$transaction([ + db.repo.update({ + where: { + id: repo.id, + }, + data: { + permittedAccounts: { + // @note: if this is a partial sync, we only want to delete the repo-driven permissions + // since we don't want to overwrite the account-driven permissions. + deleteMany: isPartialSync + ? { + source: PermissionSyncSource.REPO_DRIVEN, + } + : {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: accountIds.map((accountId) => ({ + accountId, + repoId: repo.id, + source: PermissionSyncSource.REPO_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + }, + onStarted: async ({ data: { repoId }, jobId }) => { + await db.repoPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId, logger }) => { + const { repo } = await db.repoPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + repo: { + update: { + permissionSyncedAt: new Date(), + }, + }, + }, + select: { + repo: true, + }, + }); + + logger.debug( + `Permissions synced for repo ${repo.displayName ?? repo.name}`, + ); + }, + onTerminalFailure: async ({ jobId, logger }, error) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: REPO_PERMISSION_SYNC_QUEUE.name, + }, + }); + + const { repo } = await db.repoPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + select: { + repo: true, + }, + }); + + logger.error( + `Repo permission sync job failed for repo ${repo.displayName ?? repo.name}: ${error.message}`, + ); + }, +}); + +interface ProviderPermissionSyncProps { + db: PrismaClient; + repo: RepoWithConnections; + credentials: RepoAuthCredentials; + logger: JobLogSink; +} + +interface PermissionSyncResult { + accountIds: string[]; + isPartialSync?: boolean; +} + +const getPermissionSyncResult = async ( + props: ProviderPermissionSyncProps, +): Promise => { + switch (props.repo.external_codeHostType) { + case "github": + return getGitHubPermissionSyncResult(props); + case "gitlab": + return getGitLabPermissionSyncResult(props); + case "bitbucketCloud": + return getBitbucketCloudPermissionSyncResult(props); + case "bitbucketServer": + return getBitbucketServerPermissionSyncResult(props); + default: + throw new Error( + `Unsupported code host type: ${props.repo.external_codeHostType}`, + ); + } +}; + +const getGitHubPermissionSyncResult = async ({ + db, + repo, + credentials, + logger, +}: ProviderPermissionSyncProps): Promise => { + const isGitHubCloud = credentials.hostUrl + ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME + : true; + const { octokit } = await createOctokitFromToken({ + token: credentials.token, + url: isGitHubCloud ? undefined : credentials.hostUrl, + }); + + // @note: this is a bit of a hack since the displayName _might_ not be set.. + // however, this property was introduced many versions ago and _should_ be set + // on each connection sync. Let's throw an error just in case. + if (!repo.displayName) { + throw new Error(`Repo ${repo.id} does not have a displayName`); + } + + const [owner, repoName] = repo.displayName.split("/"); + const collaborators = await getRepoCollaborators(owner, repoName, octokit); + const githubUserIds = collaborators.map((collaborator) => + collaborator.id.toString(), + ); + + logger.debug(`Found ${collaborators.length} collaborator(s)`, { + collaborators: collaborators.flatMap(({ email, login }) => ({email, login})), + }); + + const accounts = await db.account.findMany({ + where: { + providerType: "github", + providerAccountId: { + in: githubUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getGitLabPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const api = await createGitLabFromPersonalAccessToken({ + token: credentials.token, + url: credentials.hostUrl, + }); + + const projectId = repo.external_id; + if (!projectId) { + throw new Error(`Repo ${repo.id} does not have an external_id`); + } + + const members = await getProjectMembers(projectId, api); + const gitlabUserIds = members.map((member) => member.id.toString()); + + const accounts = await db.account.findMany({ + where: { + providerType: "gitlab", + providerAccountId: { + in: gitlabUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getBitbucketCloudPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const config = credentials.connectionConfig as + | BitbucketConnectionConfig + | undefined; + if (!config) { + throw new Error(`No connection config found for repo ${repo.id}`); + } + + const client = createBitbucketCloudClient(config.user, credentials.token); + + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketCloudMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketCloud; + if (!bitbucketCloudMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`, + ); + } + + const { workspace, repoSlug } = bitbucketCloudMetadata; + + // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* + // granted access to this repository. Users who have access via a group added to the repo, + // via project-level membership, or via a group in a project are NOT captured here. + // These users will still gain access through account-driven permission syncing, + // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before + // they see the repository in Sourcebot. + // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get + const users = await getExplicitUserPermissionsForCloudRepo( + client, + workspace, + repoSlug, + ); + const userAccountIds = users.map((user) => user.accountId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-cloud", + providerAccountId: { + in: userAccountIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + // Since we only fetch users who have been explicitly granted access to the repo, + // this is a partial sync. + isPartialSync: true, + }; +}; + +const getBitbucketServerPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketServerMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketServer; + if (!bitbucketServerMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`, + ); + } + + const { projectKey, repoSlug } = bitbucketServerMetadata; + const hostUrl = credentials.hostUrl; + + if (!hostUrl) { + throw new Error( + `No host URL found for Bitbucket Server repo ${repo.id}`, + ); + } + + // @note: This covers users with direct repo-level and project-level permissions. + // Users with access only via groups are NOT captured here. Those users will + // still gain access through account-driven permission syncing. + const client = createBitbucketServerClient( + hostUrl, + /* user = */ undefined, + credentials.token, + ); + const users = await getUserPermissionsForServerRepo( + client, + projectKey, + repoSlug, + ); + const userIds = users.map((user) => user.userId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-server", + providerAccountId: { in: userIds }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + isPartialSync: true, + }; +}; diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts deleted file mode 100644 index 536f48a09..000000000 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ /dev/null @@ -1,436 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { PermissionSyncSource, PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db"; -import { createLogger, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "@sourcebot/shared"; -import { env } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { DelayedError, Job, Queue, Worker } from 'bullmq'; -import { Redis } from 'ioredis'; -import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js"; -import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getExplicitUserPermissionsForCloudRepo, getUserPermissionsForServerRepo } from "../bitbucket.js"; -import { repoMetadataSchema } from "@sourcebot/shared"; -import { Settings } from "../types.js"; -import { getAuthCredentialsForRepo, setIntervalAsync } from "../utils.js"; -import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; - -type RepoPermissionSyncJob = { - jobId: string; -} - -const QUEUE_NAME = 'repoPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; -const LOG_TAG = 'repo-permission-syncer'; - -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -export class RepoPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - // @todo: make this configurable - const thresholdDate = new Date(Date.now() - this.settings.repoDrivenPermissionSyncIntervalMs); - - const repos = await this.db.repo.findMany({ - // Repos need their permissions to be synced against the code host when... - where: { - AND: [ - // They are not public. Public repositories are always visible to all users, therefore we don't - // need to explicitly perform permission syncing for them. - // @see: packages/web/src/prisma.ts - { - isPublic: false - }, - // They belong to a code host that supports permissions syncing - { - external_codeHostType: { - in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, - } - }, - // They have at least one connection with permission enforcement enabled - { - connections: { - some: { - connection: { - enforcePermissions: true, - } - } - } - }, - // They have not been synced within the threshold date. - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ], - }, - // There aren't any active or recently failed jobs. - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - RepoPermissionSyncJobStatus.PENDING, - RepoPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: RepoPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(repos); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - private async schedulePermissionSync(repos: Repo[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoPermissionSyncJob.createManyAndReturn({ - data: repos.map(repo => ({ - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'repoPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.repo.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - if (!repo) { - throw new Error(`Repo ${id} not found`); - } - - logger.debug(`Syncing permissions for repo ${repo.displayName}...`); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - if (!credentials) { - throw new Error(`No credentials found for repo ${id}`); - } - - const { - accountIds, - isPartialSync = false, - } = await (async (): Promise<{ - accountIds: string[], - isPartialSync?: boolean - }> => { - if (repo.external_codeHostType === 'github') { - const isGitHubCloud = credentials.hostUrl ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME : true; - const { octokit } = await createOctokitFromToken({ - token: credentials.token, - url: isGitHubCloud ? undefined : credentials.hostUrl, - }); - - // @note: this is a bit of a hack since the displayName _might_ not be set.. - // however, this property was introduced many versions ago and _should_ be set - // on each connection sync. Let's throw an error just in case. - if (!repo.displayName) { - throw new Error(`Repo ${id} does not have a displayName`); - } - - const [owner, repoName] = repo.displayName.split('/'); - - const collaborators = await getRepoCollaborators(owner, repoName, octokit); - const githubUserIds = collaborators.map(collaborator => collaborator.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'github', - providerAccountId: { - in: githubUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'gitlab') { - const api = await createGitLabFromPersonalAccessToken({ - token: credentials.token, - url: credentials.hostUrl, - }); - - const projectId = repo.external_id; - if (!projectId) { - throw new Error(`Repo ${id} does not have an external_id`); - } - - const members = await getProjectMembers(projectId, api); - const gitlabUserIds = members.map(member => member.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'gitlab', - providerAccountId: { - in: gitlabUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'bitbucketCloud') { - const config = credentials.connectionConfig as BitbucketConnectionConfig | undefined; - if (!config) { - throw new Error(`No connection config found for repo ${id}`); - } - - const client = createBitbucketCloudClient(config.user, credentials.token); - - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketCloudMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketCloud; - if (!bitbucketCloudMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`); - } - - const { workspace, repoSlug } = bitbucketCloudMetadata; - - // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* - // granted access to this repository. Users who have access via a group added to the repo, - // via project-level membership, or via a group in a project are NOT captured here. - // These users will still gain access through user-driven syncing (accountPermissionSyncer), - // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before - // they see the repository in Sourcebot. - // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get - const users = await getExplicitUserPermissionsForCloudRepo(client, workspace, repoSlug); - const userAccountIds = users.map(u => u.accountId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-cloud', - providerAccountId: { - in: userAccountIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - // Since we only fetch users who have been explicitly granted access to the repo, - // this is a partial sync. - isPartialSync: true, - } - } else if (repo.external_codeHostType === 'bitbucketServer') { - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketServerMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketServer; - if (!bitbucketServerMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`); - } - - const { projectKey, repoSlug } = bitbucketServerMetadata; - const hostUrl = credentials.hostUrl; - - if (!hostUrl) { - throw new Error(`No host URL found for Bitbucket Server repo ${id}`); - } - - // @note: This covers users with direct repo-level and project-level permissions. - // Users with access only via groups are NOT captured here. Those users will - // still gain access through account-driven syncing (accountPermissionSyncer). - const client = createBitbucketServerClient(hostUrl, /* user = */ undefined, credentials.token); - const users = await getUserPermissionsForServerRepo(client, projectKey, repoSlug); - const userIds = users.map(u => u.userId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-server', - providerAccountId: { in: userIds }, - issuerUrl: credentials.hostUrl, - } - }); - - return { - accountIds: accounts.map(account => account.id), - isPartialSync: true, - } - } - - throw new Error(`Unsupported code host type: ${repo.external_codeHostType}`); - })(); - - await this.db.$transaction([ - this.db.repo.update({ - where: { - id: repo.id, - }, - data: { - permittedAccounts: { - // @note: if this is a partial sync, we only want to delete the repo-driven permissions - // since we don't want to overwrite the account-driven permissions. - deleteMany: isPartialSync ? { - source: PermissionSyncSource.REPO_DRIVEN, - } : {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: accountIds.map(accountId => ({ - accountId, - repoId: repo.id, - source: PermissionSyncSource.REPO_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.COMPLETED, - repo: { - update: { - permissionSyncedAt: new Date(), - } - }, - completedAt: new Date(), - }, - select: { - repo: true - } - }); - - logger.debug(`Permissions synced for repo ${repo.displayName ?? repo.name}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err.message}`; - - if (job) { - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - repo: true - }, - }); - logger.error(errorMessage(repo.displayName ?? repo.name)); - } else { - logger.error(errorMessage('unknown repo (id not found)')); - } - } -} diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index d45f24936..f53dee93a 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -16,6 +16,8 @@ import { redis } from "./redis.js"; import { createConnectionWorkload } from "./connectionWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; +import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; +import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -59,10 +61,20 @@ const repoIndexWorkload = createRepoIndexWorkload({ db: prisma, settings, }); +const accountPermissionSyncWorkload = createAccountPermissionSyncWorkload({ + db: prisma, + settings, +}); +const repoPermissionSyncWorkload = createRepoPermissionSyncWorkload({ + db: prisma, + settings, +}); jobManager.register(reconciliationWorkload); jobManager.register(connectionWorkload); jobManager.register(repoIndexWorkload); +jobManager.register(accountPermissionSyncWorkload); +jobManager.register(repoPermissionSyncWorkload); const api = new Api(promClient, jobManager.getQueues()); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 2a6f1649f..5421c8a4b 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -1,34 +1,38 @@ -import { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { ProcessContext, Workload } from './types.js'; - -const mocks = vi.hoisted(() => ({ - enqueue: vi.fn(), - producerClose: vi.fn(), - workerClose: vi.fn(), - jobLogger: { +import { Redis } from "ioredis"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { ProcessContext, Workload } from "./types.js"; + +const mocks = vi.hoisted(() => { + const jobLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), flush: vi.fn(), - }, - workers: [] as Array<{ - processor: (job: unknown) => Promise; - handlers: Map void>; - }>, -})); + }; + return { + enqueue: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + jobLogger, + createBullMQJobLogger: vi.fn(() => jobLogger), + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, + }; +}); // The module under test creates a logger at import time; stub it so importing pure helpers // has no side effects (mirrors repoIndexManager.test.ts). -vi.mock('@sourcebot/shared', () => ({ +vi.mock("@sourcebot/shared", () => ({ createLogger: vi.fn(() => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn(), })), - createBullMQJobLogger: vi.fn(() => mocks.jobLogger), + createBullMQJobLogger: mocks.createBullMQJobLogger, BullMQClient: class { enqueue = mocks.enqueue; close = mocks.producerClose; @@ -40,19 +44,22 @@ vi.mock('@sourcebot/shared', () => ({ })); // Mock the constants module directly so its env-derived cache-dir paths don't load. -vi.mock('./constants.js', () => ({ +vi.mock("./constants.js", () => ({ WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, })); -vi.mock('@sentry/node', () => ({ +vi.mock("@sentry/node", () => ({ captureException: vi.fn(), })); -vi.mock('bullmq', () => ({ +vi.mock("bullmq", () => ({ Worker: class { private readonly record: (typeof mocks.workers)[number]; - constructor(_name: string, processor: (job: unknown) => Promise) { + constructor( + _name: string, + processor: (job: unknown) => Promise, + ) { this.record = { processor, handlers: new Map() }; mocks.workers.push(this.record); } @@ -65,60 +72,67 @@ vi.mock('bullmq', () => ({ }, })); -import { BullMQJobManager, normalizeJobState, parseDuration } from './jobManager.js'; +import { + BullMQJobManager, + normalizeJobState, + parseDuration, +} from "./jobManager.js"; -describe('parseDuration', () => { +describe("parseDuration", () => { test.each([ - ['500ms', 500], - ['30s', 30_000], - ['5m', 300_000], - ['6h', 21_600_000], - ['1d', 86_400_000], - ])('parses %s', (input, expected) => { + ["500ms", 500], + ["30s", 30_000], + ["5m", 300_000], + ["6h", 21_600_000], + ["1d", 86_400_000], + ])("parses %s", (input, expected) => { expect(parseDuration(input)).toBe(expected); }); - test('trims surrounding whitespace', () => { - expect(parseDuration(' 10m ')).toBe(600_000); + test("trims surrounding whitespace", () => { + expect(parseDuration(" 10m ")).toBe(600_000); }); - test.each(['', '5', 'm', '5x', '1.5h', '-5m', '5 m'])('throws on malformed "%s"', (input) => { - expect(() => parseDuration(input)).toThrow(); - }); + test.each(["", "5", "m", "5x", "1.5h", "-5m", "5 m"])( + 'throws on malformed "%s"', + (input) => { + expect(() => parseDuration(input)).toThrow(); + }, + ); }); -describe('normalizeJobState', () => { +describe("normalizeJobState", () => { test.each([ - 'waiting', - 'active', - 'delayed', - 'completed', - 'failed', - 'paused', + "waiting", + "active", + "delayed", + "completed", + "failed", + "paused", ])('passes through "%s"', (state) => { expect(normalizeJobState(state)).toBe(state); }); - test('collapses prioritized and waiting-children to waiting', () => { - expect(normalizeJobState('prioritized')).toBe('waiting'); - expect(normalizeJobState('waiting-children')).toBe('waiting'); + test("collapses prioritized and waiting-children to waiting", () => { + expect(normalizeJobState("prioritized")).toBe("waiting"); + expect(normalizeJobState("waiting-children")).toBe("waiting"); }); - test('maps anything unrecognized to unknown', () => { - expect(normalizeJobState('something-else')).toBe('unknown'); - expect(normalizeJobState('unknown')).toBe('unknown'); + test("maps anything unrecognized to unknown", () => { + expect(normalizeJobState("something-else")).toBe("unknown"); + expect(normalizeJobState("unknown")).toBe("unknown"); }); }); const createWorkload = ( - overrides: Partial> = {}, -): Workload<'connection-sync', { repoCount: number }> => ({ + overrides: Partial> = {}, +): Workload<"connection-sync", { repoCount: number }> => ({ queueSpec: { - name: 'connection-sync', + name: "connection-sync", dedupKey: ({ connectionId }) => `connection:${connectionId}`, jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: 500, }, @@ -130,7 +144,8 @@ const createWorkload = ( const data = { connectionId: 42, orgId: 1 }; const job = { - id: 'job-1', + id: "job-1", + queueName: "connection-sync", data, attemptsMade: 2, opts: { attempts: 2 }, @@ -138,85 +153,127 @@ const job = { updateProgress: vi.fn(), }; -describe('BullMQJobManager lifecycle', () => { +describe("BullMQJobManager lifecycle", () => { beforeEach(() => { vi.clearAllMocks(); mocks.workers.length = 0; - mocks.enqueue.mockResolvedValue('job-1'); + mocks.enqueue.mockResolvedValue("job-1"); }); - test('delegates enqueueing to BullMQClient and returns its job id', async () => { + test("delegates enqueueing to BullMQClient and returns its job id", async () => { const manager = new BullMQJobManager({} as Redis); const workload = createWorkload(); manager.register(workload); - const result = await manager.trigger('connection-sync', data); + const result = await manager.trigger("connection-sync", data); - expect(result).toBe('job-1'); + expect(result).toBe("job-1"); expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); }); - test('calls onStarted before processing and onCompleted after completion', async () => { + test("calls onStarted before processing and onCompleted after completion", async () => { const calls: string[] = []; const workload = createWorkload({ - onStarted: vi.fn(async () => { calls.push('started'); }), + onStarted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle started"); + calls.push("started"); + }), process: vi.fn(async () => { - calls.push('processed'); + calls.push("processed"); return { repoCount: 3 }; }), - onCompleted: vi.fn(async () => { calls.push('completed'); }), + onCompleted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle completed"); + calls.push("completed"); + }), }); const manager = new BullMQJobManager({} as Redis); manager.register(workload); await manager.start(); - const result = await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - expect(calls).toEqual(['started', 'processed']); + const result = await mocks.workers[0].processor({ + ...job, + attemptsMade: 0, + }); + expect(calls).toEqual(["started", "processed"]); - mocks.workers[0].handlers.get('completed')?.(job, result); - await vi.waitFor(() => expect(calls).toEqual(['started', 'processed', 'completed'])); + mocks.workers[0].handlers.get("completed")?.(job, result); + await vi.waitFor(() => + expect(calls).toEqual(["started", "processed", "completed"]), + ); expect(workload.onCompleted).toHaveBeenCalledWith( - expect.objectContaining({ data, jobId: 'job-1', maxAttempts: 2 }), + expect.objectContaining({ + data, + jobId: "job-1", + maxAttempts: 2, + logger: mocks.jobLogger, + }), { repoCount: 3 }, ); - expect(mocks.jobLogger.flush).toHaveBeenCalled(); + expect(mocks.jobLogger.info).toHaveBeenCalledWith("Lifecycle started"); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Lifecycle completed", + ); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledTimes(2); + }); }); - test('provides the structured job logger to the workload processor', async () => { - const process = vi.fn(async (context: ProcessContext<'connection-sync'>) => { - context.logger.info('Processing connection'); - return { repoCount: 3 }; - }); + test("provides the structured job logger to the workload processor", async () => { + const process = vi.fn( + async (context: ProcessContext<"connection-sync">) => { + context.logger.info("Processing connection"); + return { repoCount: 3 }; + }, + ); const manager = new BullMQJobManager({} as Redis); manager.register(createWorkload({ process })); await manager.start(); await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); - expect(process).toHaveBeenCalledWith(expect.objectContaining({ - logger: mocks.jobLogger, - })); - expect(mocks.jobLogger.info).toHaveBeenCalledWith('Processing connection'); + expect(process).toHaveBeenCalledWith( + expect.objectContaining({ + logger: mocks.jobLogger, + }), + ); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Processing connection", + ); expect(mocks.jobLogger.flush).toHaveBeenCalled(); }); - test('reports lifecycle metadata after terminal failure', async () => { - const onTerminalFailure = vi.fn(); + test("reports lifecycle metadata after terminal failure", async () => { + const onTerminalFailure = vi.fn(async ({ logger }) => { + logger.error("Lifecycle failed"); + }); const workload = createWorkload({ onTerminalFailure }); const manager = new BullMQJobManager({} as Redis); manager.register(workload); await manager.start(); - const error = new Error('failed'); - mocks.workers[0].handlers.get('failed')?.(job, error); + const error = new Error("failed"); + mocks.workers[0].handlers.get("failed")?.(job, error); await vi.waitFor(() => { - expect(onTerminalFailure).toHaveBeenCalledWith(expect.objectContaining({ - data, - jobId: 'job-1', - attemptsMade: 2, - maxAttempts: 2, - }), error); + expect(onTerminalFailure).toHaveBeenCalledWith( + expect.objectContaining({ + data, + jobId: "job-1", + attemptsMade: 2, + maxAttempts: 2, + logger: mocks.jobLogger, + }), + error, + ); }); + expect(mocks.jobLogger.error).toHaveBeenCalledWith("Lifecycle failed"); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledOnce(); + }); + expect(mocks.createBullMQJobLogger).toHaveBeenCalledWith( + expect.objectContaining({ id: "job-1", attemptsMade: 2 }), + expect.objectContaining({ attempt: 2 }), + ); }); }); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 97c44da09..a617a5812 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -1,12 +1,25 @@ import * as Sentry from "@sentry/node"; -import { BullMQClient, createBullMQJobLogger, createLogger, DataOf, JobLifecycleContext, QueueName } from "@sourcebot/shared"; +import { + BullMQClient, + createBullMQJobLogger, + createLogger, + DataOf, + JobLogSink, + QueueName, +} from "@sourcebot/shared"; import { Job, Queue, Worker } from "bullmq"; import { Redis } from "ioredis"; import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { JobDetail, JobManager, Schedule, Workload } from "./types.js"; +import { + JobDetail, + JobManager, + Schedule, + JobLifecycleContext, + Workload, +} from "./types.js"; import { prisma } from "./prisma.js"; -const LOG_TAG = 'job-manager'; +const LOG_TAG = "job-manager"; const logger = createLogger(LOG_TAG); const DURATION_UNITS_MS: Record = { @@ -20,39 +33,46 @@ const DURATION_UNITS_MS: Record = { export const parseDuration = (value: string): number => { const match = /^(\d+)(ms|s|m|h|d)$/.exec(value.trim()); if (!match) { - throw new Error(`Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`); + throw new Error( + `Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`, + ); } return Number(match[1]) * DURATION_UNITS_MS[match[2]]; }; -export const normalizeJobState = (state: string): JobDetail['state'] => { +export const normalizeJobState = (state: string): JobDetail["state"] => { switch (state) { - case 'waiting': - case 'active': - case 'delayed': - case 'completed': - case 'failed': - case 'paused': + case "waiting": + case "active": + case "delayed": + case "completed": + case "failed": + case "paused": return state; - case 'prioritized': - case 'waiting-children': - return 'waiting'; + case "prioritized": + case "waiting-children": + return "waiting"; default: - return 'unknown'; + return "unknown"; } }; const scheduleToRepeat = (schedule: Schedule) => - 'pattern' in schedule ? { pattern: schedule.pattern } : { every: parseDuration(schedule.every) }; + "pattern" in schedule + ? { pattern: schedule.pattern } + : { every: parseDuration(schedule.every) }; export class BullMQJobManager implements JobManager { - private readonly workloads = new Map>(); + private readonly workloads = new Map< + string, + Workload + >(); private readonly workers = new Map(); private readonly bullmqClient: BullMQClient; private readonly abortController = new AbortController(); constructor(private readonly connection: Redis) { - this.bullmqClient = new BullMQClient(connection, prisma); + this.bullmqClient = new BullMQClient(connection); } register(workload: Workload): void { @@ -71,7 +91,9 @@ export class BullMQJobManager implements JobManager { async start(): Promise { if (this.workloads.size === 0) { - logger.debug('start() called with nothing registered; nothing to do'); + logger.debug( + "start() called with nothing registered; nothing to do", + ); return; } @@ -80,17 +102,21 @@ export class BullMQJobManager implements JobManager { } logger.info( - `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(', ')}]`, + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(", ")}]`, ); } async trigger( workloadName: TName, - data: DataOf + data: DataOf, ): Promise { - const workload = this.workloads.get(workloadName) as Workload | undefined; + const workload = this.workloads.get(workloadName) as + | Workload + | undefined; if (!workload) { - throw new Error(`Cannot trigger unknown workload "${workloadName}"`); + throw new Error( + `Cannot trigger unknown workload "${workloadName}"`, + ); } return this.bullmqClient.enqueue(workload.queueSpec, data); } @@ -98,19 +124,25 @@ export class BullMQJobManager implements JobManager { async stop(): Promise { this.abortController.abort(); - await Promise.all([...this.workers.values()].map((worker) => - Promise.race([ - worker.close(), - new Promise((resolve) => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)), - ]), - )); + await Promise.all( + [...this.workers.values()].map((worker) => + Promise.race([ + worker.close(), + new Promise((resolve) => + setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS), + ), + ]), + ), + ); await this.bullmqClient.close(); - logger.info('Job manager stopped'); + logger.info("Job manager stopped"); } - private async startWorkload(workload: Workload): Promise { + private async startWorkload( + workload: Workload, + ): Promise { const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; const queue = this.bullmqClient.getQueue(spec); @@ -118,24 +150,29 @@ export class BullMQJobManager implements JobManager { const worker = new Worker( spec.name, async (job) => { - const jobLogger = createBullMQJobLogger( + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${spec.name}:job:${job.id ?? "unknown"}`, + }); + const lifecycleContext = this.jobLifecycleContext( job, - `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`, + jobLogger, ); - const lifecycleContext = this.jobLifecycleContext(job); try { await workload.onStarted?.(lifecycleContext); const result = await workload.process({ ...lifecycleContext, signal: this.abortController.signal, - logger: jobLogger, - updateProgress: (progress) => job.updateProgress(progress), + updateProgress: (progress) => + job.updateProgress(progress), trigger: (target, data) => this.trigger(target, data), }); return result; } catch (error) { - jobLogger.error(`Workload "${spec.name}" attempt failed`, error); + jobLogger.error( + `Workload "${spec.name}" attempt failed`, + error, + ); throw error; } finally { await jobLogger.flush(); @@ -146,18 +183,23 @@ export class BullMQJobManager implements JobManager { concurrency, maxStalledCount: 1, ...(rateLimit - ? { limiter: { max: rateLimit.max, duration: parseDuration(rateLimit.per) } } + ? { + limiter: { + max: rateLimit.max, + duration: parseDuration(rateLimit.per), + }, + } : {}), }, ); - worker.on('failed', (job, error) => { + worker.on("failed", (job, error) => { void this.onWorkloadJobFailed(workload, job, error); }); - worker.on('completed', (job, result) => { + worker.on("completed", (job, result) => { void this.onWorkloadJobCompleted(workload, job, result); }); - worker.on('error', (error) => { + worker.on("error", (error) => { logger.error(`Worker "${spec.name}" error:`, error); }); @@ -176,7 +218,9 @@ export class BullMQJobManager implements JobManager { name: spec.name, opts: { attempts: spec.jobOptions.attempts, - removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnComplete: { + count: spec.jobOptions.keep.completed, + }, removeOnFail: { count: spec.jobOptions.keep.failed }, keepLogs: spec.jobOptions.keepLogs, }, @@ -196,16 +240,36 @@ export class BullMQJobManager implements JobManager { const maxAttempts = job.opts.attempts ?? 1; const isTerminal = job.attemptsMade >= maxAttempts; if (!isTerminal) { - logger.warn(`Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`); + logger.warn( + `Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`, + ); return; } - logger.error(`Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`); + logger.error( + `Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`, + ); + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); try { - await workload.onTerminalFailure?.(this.jobLifecycleContext(job), error); + await workload.onTerminalFailure?.( + this.jobLifecycleContext(job, jobLogger), + error, + ); } catch (hookError) { Sentry.captureException(hookError); - logger.error(`onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, hookError); + jobLogger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); } } @@ -214,21 +278,41 @@ export class BullMQJobManager implements JobManager { job: Job, result: TResult, ): Promise { + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); try { - await workload.onCompleted?.(this.jobLifecycleContext(job), result); + await workload.onCompleted?.( + this.jobLifecycleContext(job, jobLogger), + result, + ); } catch (hookError) { Sentry.captureException(hookError); - logger.error(`onCompleted for workload "${workload.queueSpec.name}" threw:`, hookError); + jobLogger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); } } - private jobLifecycleContext(job: Job): JobLifecycleContext { + private jobLifecycleContext( + job: Job, + logger: JobLogSink, + ): JobLifecycleContext { return { data: job.data, - jobId: job.id ?? '', + jobId: job.id ?? "", attemptsMade: job.attemptsMade, maxAttempts: job.opts.attempts ?? 1, prisma, + logger, }; } } diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts index eadb37b88..2391cc6ee 100644 --- a/packages/backend/src/reconciliationWorkload.test.ts +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -2,7 +2,27 @@ import type { PrismaClient } from '@sourcebot/db'; import type { JobLogger } from '@sourcebot/shared'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +const mocks = vi.hoisted(() => ({ + env: { + PERMISSION_SYNC_ENABLED: 'true', + }, + hasEntitlement: vi.fn(), +})); + vi.mock('@sourcebot/shared', () => ({ + env: mocks.env, + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ + 'github', + 'gitlab', + 'bitbucket-cloud', + 'bitbucket-server', + ], + PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: [ + 'github', + 'gitlab', + 'bitbucketCloud', + 'bitbucketServer', + ], RECONCILIATION_QUEUE: { name: 'reconciliation', jobOptions: { @@ -14,12 +34,18 @@ vi.mock('@sourcebot/shared', () => ({ }, })); +vi.mock('./entitlements.js', () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + import { createReconciliationWorkload } from './reconciliationWorkload.js'; const settings = { resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, reindexIntervalMs: 60 * 60 * 1000, repoGarbageCollectionGracePeriodMs: 10 * 1000, + userDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, + repoDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, }; const logger = { @@ -33,6 +59,7 @@ const logger = { describe('reconciliationWorkload', () => { const connectionFindMany = vi.fn(); const repoFindMany = vi.fn(); + const accountFindMany = vi.fn(); const db = { connection: { findMany: connectionFindMany, @@ -40,6 +67,9 @@ describe('reconciliationWorkload', () => { repo: { findMany: repoFindMany, }, + account: { + findMany: accountFindMany, + }, } as unknown as PrismaClient; beforeEach(() => { @@ -47,6 +77,9 @@ describe('reconciliationWorkload', () => { vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); connectionFindMany.mockReset().mockResolvedValue([]); repoFindMany.mockReset().mockResolvedValue([]); + accountFindMany.mockReset().mockResolvedValue([]); + mocks.env.PERMISSION_SYNC_ENABLED = 'true'; + mocks.hasEntitlement.mockReset().mockResolvedValue(false); }); afterEach(() => { @@ -173,4 +206,170 @@ describe('reconciliationWorkload', () => { type: 'CLEANUP', }); }); + + test('submits due accounts for permission syncing when entitled', async () => { + mocks.hasEntitlement.mockResolvedValue(true); + accountFindMany.mockResolvedValue([ + { id: 'account_1' }, + { id: 'account_2' }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(accountFindMany).toHaveBeenCalledWith({ + where: { + AND: [ + { + providerType: { + in: ['github', 'gitlab', 'bitbucket-cloud', 'bitbucket-server'], + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + expect(trigger).toHaveBeenCalledWith('account-permission-sync', { + accountId: 'account_1', + }); + expect(trigger).toHaveBeenCalledWith('account-permission-sync', { + accountId: 'account_2', + }); + }); + + test('does not query accounts without the permission syncing entitlement', async () => { + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(accountFindMany).not.toHaveBeenCalled(); + expect(repoFindMany).toHaveBeenCalledTimes(2); + }); + + test('does not query accounts when permission syncing is disabled', async () => { + mocks.env.PERMISSION_SYNC_ENABLED = 'false'; + mocks.hasEntitlement.mockResolvedValue(true); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(mocks.hasEntitlement).not.toHaveBeenCalled(); + expect(accountFindMany).not.toHaveBeenCalled(); + expect(repoFindMany).toHaveBeenCalledTimes(2); + }); + + test('submits due private repos for permission syncing when entitled', async () => { + mocks.hasEntitlement.mockResolvedValue(true); + repoFindMany + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { id: 42 }, + { id: 84 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(repoFindMany).toHaveBeenNthCalledWith(3, { + where: { + AND: [ + { + isPublic: false, + }, + { + external_codeHostType: { + in: ['github', 'gitlab', 'bitbucketCloud', 'bitbucketServer'], + }, + }, + { + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { + repoId: 42, + }); + expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { + repoId: 84, + }); + }); }); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts index 90a14f14e..fa2f29cfa 100644 --- a/packages/backend/src/reconciliationWorkload.ts +++ b/packages/backend/src/reconciliationWorkload.ts @@ -1,5 +1,6 @@ import { PrismaClient } from "@sourcebot/db"; -import { RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { env, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { hasEntitlement } from "./entitlements.js"; import { Settings, Workload } from "./types.js"; interface ReconciliationWorkloadDependencies { @@ -15,72 +16,153 @@ export const createReconciliationWorkload = ({ schedule: { every: '10s' }, queueSpec: RECONCILIATION_QUEUE, process: async ({ logger, trigger }) => { - const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); - const connections = await db.connection.findMany({ - where: { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: connectionThreshold } }, - ], - }, - select: { - id: true, - orgId: true, - }, - }); + // Connections + { + const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connections = await db.connection.findMany({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: connectionThreshold } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + + await Promise.all(connections.map(async (connection) => { + await trigger('connection-sync', { + connectionId: connection.id, + orgId: connection.orgId, + }); + })); + } - await Promise.all(connections.map(async (connection) => { - logger.debug(`Scheduling connection sync for connection ${connection.id}`); - await trigger('connection-sync', { - connectionId: connection.id, - orgId: connection.orgId, + // Repo garbage collection + { + const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); + const reposToCleanup = await db.repo.findMany({ + where: { + connections: { + none: {}, + }, + isAutoCleanupDisabled: false, + OR: [ + { indexedAt: null }, + { indexedAt: { lt: cleanupThreshold } }, + ], + }, + select: { + id: true, + }, }); - })); - const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); - const reposToCleanup = await db.repo.findMany({ - where: { - connections: { - none: {}, + await Promise.all(reposToCleanup.map(async ({ id }) => { + logger.debug(`Scheduling cleanup for repo ${id}`); + await trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }); + })); + } + + // Repo indexing + { + const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); + const reposToIndex = await db.repo.findMany({ + where: { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: indexThreshold } }, + ], + }, + select: { + id: true, }, - isAutoCleanupDisabled: false, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: cleanupThreshold } }, - ], - }, - select: { - id: true, - }, - }); + }); + + await Promise.all(reposToIndex.map(async ({ id }) => { + await trigger('repo-index', { + repoId: id, + type: 'INDEX', + }); + })); + } - await Promise.all(reposToCleanup.map(async ({ id }) => { - logger.debug(`Scheduling cleanup for repo ${id}`); - await trigger('repo-index', { - repoId: id, - type: 'CLEANUP', + // Permission syncing + if ( + env.PERMISSION_SYNC_ENABLED === 'true' && + await hasEntitlement('permission-syncing') + ) { + const accountThreshold = new Date(Date.now() - settings.userDrivenPermissionSyncIntervalMs); + const accounts = await db.account.findMany({ + where: { + AND: [ + { + providerType: { + in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: accountThreshold } }, + ], + }, + ], + }, + select: { + id: true, + }, }); - })); - const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); - const reposToIndex = await db.repo.findMany({ - where: { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: indexThreshold } }, - ], - }, - select: { - id: true, - }, - }); + await Promise.all(accounts.map(async ({ id }) => { + await trigger('account-permission-sync', { + accountId: id, + }); + })); - await Promise.all(reposToIndex.map(async ({ id }) => { - logger.debug(`Scheduling index for repo ${id}`); - await trigger('repo-index', { - repoId: id, - type: 'INDEX', + const repoThreshold = new Date(Date.now() - settings.repoDrivenPermissionSyncIntervalMs); + const repos = await db.repo.findMany({ + where: { + AND: [ + { + isPublic: false, + }, + { + external_codeHostType: { + in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, + }, + }, + { + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: repoThreshold } }, + ], + }, + ], + }, + select: { + id: true, + }, }); - })); + + await Promise.all(repos.map(async ({ id }) => { + await trigger('repo-permission-sync', { + repoId: id, + }); + })); + } }, }); diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index 076814b3d..023a71e07 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -1,19 +1,21 @@ -import type { PrismaClient } from '@sourcebot/db'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { createRepoIndexWorkload } from './repoIndexWorkload.js'; +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; const repoIndexingJobUpsert = vi.fn(); const repoIndexingJobUpdate = vi.fn(); const repoUpdate = vi.fn(); -const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ - repoIndexingJob: { - upsert: repoIndexingJobUpsert, - update: repoIndexingJobUpdate, - }, - repo: { - update: repoUpdate, - }, -})); +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => + callback({ + repoIndexingJob: { + upsert: repoIndexingJobUpsert, + update: repoIndexingJobUpdate, + }, + repo: { + update: repoUpdate, + }, + }), +); const db = { $transaction: transaction, @@ -26,45 +28,53 @@ const workload = createRepoIndexWorkload({ } as never, }); +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + const lifecycleContext = { data: { repoId: 42, - type: 'INDEX' as const, + type: "INDEX" as const, }, - jobId: 'job-1', + jobId: "job-1", attemptsMade: 0, maxAttempts: 2, prisma: db, + logger: lifecycleLogger, }; -describe('repoIndexWorkload lifecycle', () => { +describe("repoIndexWorkload lifecycle", () => { beforeEach(() => { vi.clearAllMocks(); }); - test('declares database-backed lifecycle hooks', () => { - expect(workload.onStarted).toBeTypeOf('function'); - expect(workload.onCompleted).toBeTypeOf('function'); - expect(workload.onTerminalFailure).toBeTypeOf('function'); + test("declares database-backed lifecycle hooks", () => { + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); }); - test('marks the repo indexing job and repo as in progress when started', async () => { + test("marks the repo indexing job and repo as in progress when started", async () => { await workload.onStarted?.(lifecycleContext); expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, update: { - status: 'IN_PROGRESS', + status: "IN_PROGRESS", completedAt: null, errorMessage: null, }, create: { - id: 'job-1', + id: "job-1", repoId: 42, - type: 'INDEX', - status: 'IN_PROGRESS', + type: "INDEX", + status: "IN_PROGRESS", }, }); expect(repoUpdate).toHaveBeenCalledWith({ @@ -72,20 +82,20 @@ describe('repoIndexWorkload lifecycle', () => { id: 42, }, data: { - latestIndexingJobStatus: 'IN_PROGRESS', + latestIndexingJobStatus: "IN_PROGRESS", }, }); }); - test('marks the repo indexing job and repo as completed', async () => { + test("marks the repo indexing job and repo as completed", async () => { await workload.onCompleted?.(lifecycleContext, undefined); expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'COMPLETED', + status: "COMPLETED", completedAt: expect.any(Date), errorMessage: null, }, @@ -95,37 +105,40 @@ describe('repoIndexWorkload lifecycle', () => { id: 42, }, data: { - latestIndexingJobStatus: 'COMPLETED', + latestIndexingJobStatus: "COMPLETED", }, }); }); - test('does not update a completed cleanup job after its repo cascades the job row', async () => { - await workload.onCompleted?.({ - ...lifecycleContext, - data: { - repoId: 42, - type: 'CLEANUP', + test("does not update a completed cleanup job after its repo cascades the job row", async () => { + await workload.onCompleted?.( + { + ...lifecycleContext, + data: { + repoId: 42, + type: "CLEANUP", + }, }, - }, undefined); + undefined, + ); expect(transaction).not.toHaveBeenCalled(); }); - test('marks the repo indexing job and repo as failed after terminal failure', async () => { + test("marks the repo indexing job and repo as failed after terminal failure", async () => { await workload.onTerminalFailure?.( lifecycleContext, - new Error('Unable to clone repository'), + new Error("Unable to clone repository"), ); expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ where: { - id: 'job-1', + id: "job-1", }, data: { - status: 'FAILED', + status: "FAILED", completedAt: expect.any(Date), - errorMessage: 'Unable to clone repository', + errorMessage: "Unable to clone repository", }, }); expect(repoUpdate).toHaveBeenCalledWith({ @@ -133,7 +146,7 @@ describe('repoIndexWorkload lifecycle', () => { id: 42, }, data: { - latestIndexingJobStatus: 'FAILED', + latestIndexingJobStatus: "FAILED", }, }); }); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 848d9f48f..1cc29e147 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,19 +1,28 @@ -import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; +import { + Connection, + PrismaClient, + Repo, + RepoToConnection, +} from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; -import { DataOf, JobLifecycleContext, JobLogger, QueueName, QueueSpec } from "@sourcebot/shared"; +import { DataOf, JobLogSink, QueueName, QueueSpec } from "@sourcebot/shared"; export type Settings = Required; // @see : https://stackoverflow.com/a/61132308 -export type DeepPartial = T extends object ? { - [P in keyof T]?: DeepPartial; -} : T; +export type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial; + } + : T; // @see: https://stackoverflow.com/a/69328045 export type WithRequired = T & { [P in K]-?: T[P] }; -export type RepoWithConnections = Repo & { connections: (RepoToConnection & { connection: Connection })[] }; +export type RepoWithConnections = Repo & { + connections: (RepoToConnection & { connection: Connection })[]; +}; export type RepoAuthCredentials = { hostUrl?: string; @@ -21,12 +30,20 @@ export type RepoAuthCredentials = { cloneUrlWithToken?: string; authHeader?: string; connectionConfig?: ConnectionConfig; -} +}; +export interface JobLifecycleContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; + prisma: PrismaClient; + logger: JobLogSink; +} -export interface ProcessContext extends JobLifecycleContext { +export interface ProcessContext + extends JobLifecycleContext { signal: AbortSignal; - logger: JobLogger; updateProgress(progress: number | object): Promise; trigger(workload: T, data: DataOf): Promise; } @@ -56,9 +73,15 @@ export interface Workload { /** Called before `process` on every attempt. */ onStarted?(ctx: JobLifecycleContext): Promise; /** Called after BullMQ marks the job as completed. */ - onCompleted?(ctx: JobLifecycleContext, result: TResult): Promise; + onCompleted?( + ctx: JobLifecycleContext, + result: TResult, + ): Promise; /** Called after BullMQ exhausts all attempts and marks the job as failed. */ - onTerminalFailure?(ctx: JobLifecycleContext, err: Error): Promise; + onTerminalFailure?( + ctx: JobLifecycleContext, + err: Error, + ): Promise; } export interface JobManager { @@ -69,12 +92,10 @@ export interface JobManager { trigger( workload: TName, - data: DataOf + data: DataOf, ): Promise; } - - export interface QueueCounts { waiting: number; active: number; @@ -83,13 +104,20 @@ export interface QueueCounts { failed: number; paused: number; prioritized?: number; - 'waiting-children'?: number; + "waiting-children"?: number; } export interface JobDetail { id: string; name: string; - state: 'waiting' | 'active' | 'delayed' | 'completed' | 'failed' | 'paused' | 'unknown'; + state: + | "waiting" + | "active" + | "delayed" + | "completed" + | "failed" + | "paused" + | "unknown"; data: TData; attemptsMade: number; maxAttempts: number; diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index 15296f33a..a15b31ea3 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -41,17 +41,17 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -95,12 +95,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, @@ -227,17 +227,17 @@ const schema = { }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -281,12 +281,12 @@ const schema = { }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } }, diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 901e7762b..c18bc9301 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -110,16 +110,16 @@ export interface Settings { */ reindexRepoPollingIntervalMs?: number; /** - * The number of connection sync jobs to run concurrently. Defaults to 2. + * The number of connection sync jobs to run concurrently. Defaults to 8. */ maxConnectionSyncJobConcurrency?: number; /** - * The number of repo indexing jobs to run concurrently. Defaults to 2. + * The number of repo indexing jobs to run concurrently. Defaults to 8. */ maxRepoIndexingJobConcurrency?: number; /** * @deprecated - * The number of repo GC jobs to run concurrently. Defaults to 2. + * The number of repo GC jobs to run concurrently. Defaults to 8. */ maxRepoGarbageCollectionJobConcurrency?: number; /** @@ -154,11 +154,11 @@ export interface Settings { */ experiment_userDrivenPermissionSyncIntervalMs?: number; /** - * The number of account permission sync jobs to run concurrently. Defaults to 2. + * The number of account permission sync jobs to run concurrently. Defaults to 8. */ maxAccountPermissionSyncJobConcurrency?: number; /** - * The number of repo permission sync jobs to run concurrently. Defaults to 2. + * The number of repo permission sync jobs to run concurrently. Defaults to 8. */ maxRepoPermissionSyncJobConcurrency?: number; } diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts deleted file mode 100644 index 34955363c..000000000 --- a/packages/shared/src/bullmqClient.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -import type { Redis } from 'ioredis'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; -import type { QueueSpec } from './queue.js'; -import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from './jobLogger.js'; -import { CONNECTION_QUEUE, REPO_INDEX_QUEUE } from './queue.js'; - -const queueMocks = vi.hoisted(() => ({ - add: vi.fn(), - close: vi.fn(), - getJob: vi.fn(), - getJobLogs: vi.fn(), -})); - -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('./logger.js', () => ({ - createLogger: vi.fn(() => ({ - error: vi.fn(), - })), -})); - -vi.mock('bullmq', () => ({ - Queue: class { - constructor() { - return queueMocks; - } - }, -})); - -import { BullMQClient } from './bullmqClient.js'; -import { PrismaClient } from '@sourcebot/db'; - -const connectionSpec: QueueSpec<'connection-sync'> = { - name: 'connection-sync', - dedupKey: ({ connectionId }) => `connection:${connectionId}`, - jobOptions: { - attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, - keep: { completed: 50, failed: 50 }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }, -}; - -const data = { connectionId: 42, orgId: 1 }; - -describe('BullMQClient', () => { - const redis = {} as Redis; - const prisma = {} as PrismaClient; - - beforeEach(() => { - vi.clearAllMocks(); - queueMocks.add.mockImplementation(async (_name, _data, options) => ({ id: options.jobId })); - }); - - test('returns the job id when BullMQ accepts the proposed id', async () => { - const client = new BullMQClient(redis, prisma); - - const result = await client.enqueue(connectionSpec, data); - - expect(result).toEqual(expect.any(String)); - expect(queueMocks.add).toHaveBeenCalledWith( - 'connection-sync', - data, - expect.objectContaining({ - jobId: result, - deduplication: { id: 'connection:42' }, - keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, - }), - ); - }); - - test('calls onEnqueued when a new job is created', async () => { - const onEnqueued = vi.fn(); - const client = new BullMQClient(redis, prisma); - - const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(onEnqueued).toHaveBeenCalledWith({ - data, - jobId: result, - attemptsMade: 0, - maxAttempts: 2, - prisma, - }); - }); - - test('returns the existing job id without calling onEnqueued when BullMQ deduplicates the enqueue', async () => { - const onEnqueued = vi.fn(); - queueMocks.add.mockResolvedValue({ id: 'existing-job' }); - const client = new BullMQClient(redis, prisma); - - const result = await client.enqueue({ ...connectionSpec, onEnqueued }, data); - - expect(result).toBe('existing-job'); - expect(onEnqueued).not.toHaveBeenCalled(); - }); - - test('upserts a pending connection sync job when enqueueing a connection sync', async () => { - const connectionSyncJobUpsert = vi.fn(); - const client = new BullMQClient(redis, { - connectionSyncJob: { - upsert: connectionSyncJobUpsert, - }, - } as unknown as PrismaClient); - - const result = await client.enqueue(CONNECTION_QUEUE, data); - - expect(connectionSyncJobUpsert).toHaveBeenCalledWith({ - where: { - id: result, - }, - update: {}, - create: { - id: result, - connectionId: 42, - status: 'PENDING', - warningMessages: [], - }, - }); - }); - - test('upserts a pending repo indexing job when enqueueing with repo-level deduplication', async () => { - const repoIndexingJobUpsert = vi.fn(); - const client = new BullMQClient(redis, { - repoIndexingJob: { - upsert: repoIndexingJobUpsert, - }, - } as unknown as PrismaClient); - - const result = await client.enqueue(REPO_INDEX_QUEUE, { - repoId: 42, - type: 'INDEX', - }); - - expect(queueMocks.add).toHaveBeenCalledWith( - 'repo-index', - { - repoId: 42, - type: 'INDEX', - }, - expect.objectContaining({ - deduplication: { id: 'repo:42' }, - }), - ); - expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ - where: { - id: result, - }, - update: {}, - create: { - id: result, - repoId: 42, - type: 'INDEX', - status: 'PENDING', - }, - }); - }); - - test.each([ - ['waiting', 'PENDING'], - ['waiting-children', 'PENDING'], - ['delayed', 'PENDING'], - ['prioritized', 'PENDING'], - ['paused', 'PENDING'], - ['active', 'IN_PROGRESS'], - ['completed', 'COMPLETED'], - ])('maps BullMQ state %s to %s', async (state, expectedStatus) => { - queueMocks.getJob.mockResolvedValue({ - id: 'job-1', - data, - getState: vi.fn().mockResolvedValue(state), - }); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ - id: 'job-1', - data, - status: expectedStatus, - errorMessage: null, - }); - }); - - test('returns the failure reason for a failed job', async () => { - queueMocks.getJob.mockResolvedValue({ - id: 'job-1', - data, - failedReason: 'Connection credentials expired', - getState: vi.fn().mockResolvedValue('failed'), - }); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJob(connectionSpec, 'job-1')).resolves.toEqual({ - id: 'job-1', - data, - status: 'FAILED', - errorMessage: 'Connection credentials expired', - }); - }); - - test.each([ - ['missing job', undefined], - ['unknown state', { - id: 'job-1', - data, - getState: vi.fn().mockResolvedValue('unknown'), - }], - ])('returns null for a %s', async (_label, job) => { - queueMocks.getJob.mockResolvedValue(job); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJob(connectionSpec, 'job-1')).resolves.toBeNull(); - }); - - test('reads and parses incremental job logs', async () => { - queueMocks.getJobLogs.mockResolvedValue({ - logs: [ - JSON.stringify({ - version: 1, - timestamp: '2026-07-28T12:00:00.000Z', - level: 'warn', - message: 'Repository skipped', - attempt: 1, - }), - ], - count: 4, - }); - const client = new BullMQClient(redis, prisma); - - await expect(client.getJobLogs(connectionSpec, 'job-1', { - start: 3, - ascending: true, - })).resolves.toEqual({ - logs: [{ - version: 1, - timestamp: '2026-07-28T12:00:00.000Z', - level: 'warn', - message: 'Repository skipped', - attempt: 1, - }], - count: 4, - }); - expect(queueMocks.getJobLogs).toHaveBeenCalledWith( - 'job-1', - 3, - undefined, - true, - ); - }); -}); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index 52a6bff54..5e5e2e43a 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -1,15 +1,10 @@ -import * as Sentry from "@sentry/node"; import { Queue } from "bullmq"; import { randomUUID } from "crypto"; import { Redis } from "ioredis"; -import { createLogger } from "./logger.js"; import { DataOf, QueueName, QueueSpec } from "./queue.js"; -import { PrismaClient } from "@sourcebot/db"; import { readBullMQJobLogs } from "./jobLogger.js"; import type { GetJobLogsOptions, JobLogs } from "./jobLogger.js"; -const logger = createLogger('job-producer'); - export type WorkloadJobStatus = "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED"; export interface WorkloadJob { @@ -52,7 +47,6 @@ export class BullMQClient { constructor( private readonly connection: Redis, - private readonly prisma: PrismaClient, ) {} getQueue(spec: QueueSpec): WorkloadQueue { @@ -117,22 +111,6 @@ export class BullMQClient { throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); } - const isEnqueued = job.id === requestedJobId; - if (isEnqueued && spec.onEnqueued) { - try { - await spec.onEnqueued({ - data, - jobId: job.id, - attemptsMade: 0, - maxAttempts: spec.jobOptions.attempts, - prisma: this.prisma, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`onEnqueued for workload "${spec.name}" threw:`, error); - } - } - return job.id; } diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index fce116549..134774a0c 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -23,8 +23,8 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, // 24 hours resyncConnectionPollingIntervalMs: 1000 * 1, // 1 second reindexRepoPollingIntervalMs: 1000 * 1, // 1 second - maxConnectionSyncJobConcurrency: 2, - maxRepoIndexingJobConcurrency: 2, + maxConnectionSyncJobConcurrency: 8, + maxRepoIndexingJobConcurrency: 8, maxRepoGarbageCollectionJobConcurrency: 2, repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours @@ -33,8 +33,8 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, // 24 hours (deprecated) - maxAccountPermissionSyncJobConcurrency: 2, - maxRepoPermissionSyncJobConcurrency: 2, + maxAccountPermissionSyncJobConcurrency: 8, + maxRepoPermissionSyncJobConcurrency: 8, } export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: CodeHostType[] = [ diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 8e082fe63..d374e9f18 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -397,7 +397,6 @@ const options = { REDIS_TLS_HONOR_CIPHER_ORDER: booleanSchema.optional(), REDIS_TLS_KEY_PASSPHRASE: z.string().optional(), - CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 91d42e05b..d9fe61e1c 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -94,14 +94,15 @@ export { export type { Version } from "./versionUtils.js"; export type { QueueName, - JobLifecycleContext, DataOf, QueueSpec, } from "./queue.js" export { + ACCOUNT_PERMISSION_SYNC_QUEUE, CONNECTION_QUEUE, RECONCILIATION_QUEUE, REPO_INDEX_QUEUE, + REPO_PERMISSION_SYNC_QUEUE, } from "./queue.js"; export { BullMQClient, diff --git a/packages/shared/src/jobLogger.test.ts b/packages/shared/src/jobLogger.test.ts index c6435ea00..cf727c2ab 100644 --- a/packages/shared/src/jobLogger.test.ts +++ b/packages/shared/src/jobLogger.test.ts @@ -71,6 +71,25 @@ describe("createBullMQJobLogger", () => { await expect(logger.flush()).resolves.toBeUndefined(); expect(mocks.applicationError).toHaveBeenCalled(); }); + + test("uses the supplied attempt for post-processing lifecycle logs", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger( + { + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 2, + log, + }, + { attempt: 2 }, + ); + + logger.info("Completed"); + await logger.flush(); + + expect(JSON.parse(log.mock.calls[0][0])).toMatchObject({ attempt: 2 }); + }); }); describe("readBullMQJobLogs", () => { diff --git a/packages/shared/src/jobLogger.ts b/packages/shared/src/jobLogger.ts index 57df9311b..f496cdab6 100644 --- a/packages/shared/src/jobLogger.ts +++ b/packages/shared/src/jobLogger.ts @@ -37,11 +37,15 @@ export interface JobLogs { count: number; } -type BullMQLogJob = Pick; +type BullMQLogJob = Pick< + Job, + "id" | "name" | "queueName" | "attemptsMade" | "log" +>; type JobLogQueue = Pick; const JOB_LOG_LEVELS = new Set(["debug", "info", "warn", "error"]); -const SENSITIVE_FIELD_NAME = /authorization|cookie|credential|password|private.?key|secret|token/i; +const SENSITIVE_FIELD_NAME = + /authorization|cookie|credential|password|private.?key|secret|token/i; const MAX_FIELD_DEPTH = 6; const sanitizeValue = ( @@ -104,7 +108,11 @@ const sanitizeFields = (fields: unknown): JobLogFields | undefined => { } const sanitized = sanitizeValue(fields, new WeakSet(), 0); - if (sanitized !== null && typeof sanitized === "object" && !Array.isArray(sanitized)) { + if ( + sanitized !== null && + typeof sanitized === "object" && + !Array.isArray(sanitized) + ) { return sanitized as JobLogFields; } return { value: sanitized }; @@ -163,12 +171,22 @@ export const readBullMQJobLogs = async ( export const createBullMQJobLogger = ( job: BullMQLogJob, - label = `${job.queueName}:job:${job.id ?? "unknown"}`, + options: { + label?: string; + attempt?: number; + } = {}, ): JobLogger => { + const label = + options.label ?? `${job.queueName}:job:${job.id ?? "unknown"}`; + const attempt = options.attempt ?? job.attemptsMade + 1; const applicationLogger = createLogger(label); const pendingWrites = new Set>(); - const write = (level: JobLogLevel, message: string, rawFields?: unknown): void => { + const write = ( + level: JobLogLevel, + message: string, + rawFields?: unknown, + ): void => { const fields = sanitizeFields(rawFields); applicationLogger.log(level, message, fields); @@ -177,10 +195,11 @@ export const createBullMQJobLogger = ( timestamp: new Date().toISOString(), level, message, - attempt: job.attemptsMade + 1, + attempt, ...(fields ? { fields } : {}), }; - const pendingWrite = job.log(JSON.stringify(entry)) + const pendingWrite = job + .log(JSON.stringify(entry)) .then(() => undefined) .catch((error: unknown) => { applicationLogger.error( diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 7c65ac5ef..d7b8d0271 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,5 +1,3 @@ - -import { ConnectionSyncJobStatus, PrismaClient, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; export type QueueName = keyof QueueRegistry; @@ -7,75 +5,76 @@ export type DataOf = QueueRegistry[TName]; type EmptyJobData = Record; interface QueueRegistry { - 'reconciliation': EmptyJobData, - 'connection-sync': { - connectionId: number, - orgId: number - }, - 'repo-index': { - repoId: number, - type: 'INDEX' | 'CLEANUP', - }, + reconciliation: EmptyJobData; + "connection-sync": { + connectionId: number; + orgId: number; + }; + "repo-index": { + repoId: number; + type: "INDEX" | "CLEANUP"; + }; + "account-permission-sync": { + accountId: string; + }; + "repo-permission-sync": { + repoId: number; + }; } -export const RECONCILIATION_QUEUE: QueueSpec<'reconciliation'> = { - name: 'reconciliation', +export const RECONCILIATION_QUEUE: QueueSpec<"reconciliation"> = { + name: "reconciliation", jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, }; -export const CONNECTION_QUEUE: QueueSpec<'connection-sync'> = { - name: 'connection-sync', +export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { + name: "connection-sync", jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, dedupKey: (data) => `connection:${data.connectionId}`, - onEnqueued: async ({ prisma, data: { connectionId }, jobId }) => { - await prisma.connectionSyncJob.upsert({ - where: { - id: jobId, - }, - update: {}, - create: { - id: jobId, - connectionId, - status: ConnectionSyncJobStatus.PENDING, - warningMessages: [], - }, - }); - } }; -export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = { - name: 'repo-index', +export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { + name: "repo-index", jobOptions: { attempts: 2, - backoff: { type: 'exponential', delayMs: 5000 }, + backoff: { type: "exponential", delayMs: 5000 }, keep: { completed: 50, failed: 50 }, keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, dedupKey: (data) => `repo:${data.repoId}`, - onEnqueued: async ({ prisma, data: { repoId, type }, jobId }) => { - await prisma.repoIndexingJob.upsert({ - where: { - id: jobId, - }, - update: {}, - create: { - id: jobId, - repoId, - type: RepoIndexingJobType[type], - status: RepoIndexingJobStatus.PENDING, - }, - }); +}; + +export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = + { + name: "account-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + dedupKey: (data) => `account:${data.accountId}`, + }; + +export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { + name: "repo-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, }, + dedupKey: (data) => `repo:${data.repoId}`, }; export interface QueueSpec { @@ -83,17 +82,8 @@ export interface QueueSpec { dedupKey?(data: DataOf): string; jobOptions: { attempts: number; - backoff: { type: 'fixed' | 'exponential'; delayMs: number }; + backoff: { type: "fixed" | "exponential"; delayMs: number }; keep: { completed: number; failed: number }; keepLogs: number; }; - onEnqueued?(ctx: JobLifecycleContext): Promise; -} - -export interface JobLifecycleContext { - data: DataOf; - jobId: string; - attemptsMade: number; - maxAttempts: number; - prisma: PrismaClient; } diff --git a/packages/web/src/lib/bullmqClient.ts b/packages/web/src/lib/bullmqClient.ts index ef5d9524f..a891bf4bd 100644 --- a/packages/web/src/lib/bullmqClient.ts +++ b/packages/web/src/lib/bullmqClient.ts @@ -2,11 +2,10 @@ import 'server-only'; import { BullMQClient } from '@sourcebot/shared'; import { getRedisClient } from './redis'; -import { __unsafePrisma } from '@/prisma'; let client: BullMQClient | undefined; export function getBullMQClient() { - client ??= new BullMQClient(getRedisClient(), __unsafePrisma); + client ??= new BullMQClient(getRedisClient()); return client; } diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 6b5b4fbb1..21499ef0f 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -40,17 +40,17 @@ }, "maxConnectionSyncJobConcurrency": { "type": "number", - "description": "The number of connection sync jobs to run concurrently. Defaults to 2.", + "description": "The number of connection sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoIndexingJobConcurrency": { "type": "number", - "description": "The number of repo indexing jobs to run concurrently. Defaults to 2.", + "description": "The number of repo indexing jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoGarbageCollectionJobConcurrency": { "type": "number", - "description": "The number of repo GC jobs to run concurrently. Defaults to 2.", + "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", "minimum": 1, "deprecated": true }, @@ -94,12 +94,12 @@ }, "maxAccountPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of account permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of account permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 }, "maxRepoPermissionSyncJobConcurrency": { "type": "number", - "description": "The number of repo permission sync jobs to run concurrently. Defaults to 2.", + "description": "The number of repo permission sync jobs to run concurrently. Defaults to 8.", "minimum": 1 } },