diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index a9603abdc..1cff30599 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -32,7 +32,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -52,7 +53,8 @@ "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -216,7 +218,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -236,7 +239,8 @@ "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", diff --git a/packages/backend/package.json b/packages/backend/package.json index 951ef1f33..bc1fc5e03 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -22,6 +22,9 @@ "vitest": "^4.1.4" }, "dependencies": { + "@bull-board/api": "6.11.2", + "@bull-board/express": "6.11.2", + "@bull-board/ui": "6.11.2", "@coderabbitai/bitbucket": "^1.1.3", "@gitbeaker/rest": "^40.5.1", "@octokit/app": "^16.1.1", @@ -35,7 +38,7 @@ "@types/express": "^5.0.0", "argparse": "^2.0.1", "azure-devops-node-api": "^15.1.1", - "bullmq": "^5.34.10", + "bullmq": "^5.81.3", "chokidar": "^4.0.3", "cross-fetch": "^4.0.0", "dotenv": "^16.4.5", @@ -46,13 +49,12 @@ "gitea-js": "^1.22.0", "glob": "^11.1.0", "http-status-codes": "^2.3.0", - "ioredis": "^5.4.2", + "ioredis": "^5.11.1", "lowdb": "^7.0.1", "micromatch": "^4.0.8", "p-limit": "^7.2.0", "posthog-node": "^5.24.15", "prom-client": "^15.1.3", - "redlock": "5.0.0-beta.2", "simple-git": "^3.36.0", "zod": "^3.25.76" } diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index d6cc5b1db..4c829a3e9 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -1,19 +1,13 @@ -import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; -import * as Sentry from '@sentry/node'; -import { hasEntitlement } from './entitlements.js'; -import { createLogger, doesIdpSupportPermissionSyncing, env } from '@sourcebot/shared'; +import { createLogger, env } from '@sourcebot/shared'; +import { createBullBoard } from '@bull-board/api'; +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js'; +import { ExpressAdapter } from '@bull-board/express'; +import { Queue } from 'bullmq'; import express, { NextFunction, Request, Response } from 'express'; import 'express-async-errors'; import * as http from "http"; -import { ConnectionManager } from './connectionManager.js'; -import { AccountPermissionSyncer } from './ee/accountPermissionSyncer.js'; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from './repoIndexManager.js'; -import { createGitHubRepoRecord } from './repoCompileUtils.js'; -import { isNotFound } from './errors.js'; -import { Octokit } from '@octokit/rest'; -import { SINGLE_TENANT_ORG_ID } from './constants.js'; -import z from 'zod'; +import * as Sentry from "@sentry/node"; const logger = createLogger('api'); @@ -23,17 +17,19 @@ const PORT = Number(workerApiUrl.port) || (workerApiUrl.protocol === "https:" ? export class Api { private server: http.Server; - constructor( - promClient: PromClient, - private prisma: PrismaClient, - private connectionManager: ConnectionManager, - private repoIndexManager: RepoIndexManager, - private accountPermissionSyncer: AccountPermissionSyncer, - ) { + constructor(promClient: PromClient, queues: Queue[]) { const app = express(); app.use(express.json()); app.use(express.urlencoded({ extended: true })); + const bullBoardAdapter = new ExpressAdapter(); + bullBoardAdapter.setBasePath('/admin/queues'); + createBullBoard({ + queues: queues.map(queue => new BullMQAdapter(queue, { readOnlyMode: true })), + serverAdapter: bullBoardAdapter, + }); + app.use('/admin/queues', bullBoardAdapter.getRouter()); + // Prometheus metrics endpoint app.use('/metrics', async (_req: Request, res: Response) => { res.set('Content-Type', promClient.registry.contentType); @@ -41,11 +37,6 @@ export class Api { res.end(metrics); }); - app.post('/api/sync-connection', this.syncConnection.bind(this)); - app.post('/api/index-repo', this.indexRepo.bind(this)); - app.post('/api/trigger-account-permission-sync', this.triggerAccountPermissionSync.bind(this)); - app.post(`/api/experimental/add-github-repo`, this.experimental_addGithubRepo.bind(this)); - app.use((error: unknown, _req: Request, _res: Response, next: NextFunction) => { Sentry.captureException(error); next(error); @@ -53,147 +44,10 @@ export class Api { this.server = app.listen(PORT, () => { logger.debug(`API server is running on port ${PORT}`); + logger.debug(`Bull Board is available at ${workerApiUrl.origin}/admin/queues`); }); } - private async syncConnection(req: Request, res: Response) { - const schema = z.object({ - connectionId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { connectionId } = parsed.data; - const connection = await this.prisma.connection.findUnique({ - where: { - id: connectionId, - } - }); - - if (!connection) { - res.status(404).json({ error: 'Connection not found' }); - return; - } - - const [jobId] = await this.connectionManager.createJobs([connection]); - - res.status(200).json({ jobId }); - } - - private async indexRepo(req: Request, res: Response) { - const schema = z.object({ - repoId: z.number(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { repoId } = parsed.data; - const repo = await this.prisma.repo.findUnique({ - where: { id: repoId }, - }); - - if (!repo) { - res.status(404).json({ error: 'Repo not found' }); - return; - } - - const [jobId] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); - res.status(200).json({ jobId }); - } - - private async triggerAccountPermissionSync(req: Request, res: Response) { - if (env.PERMISSION_SYNC_ENABLED !== 'true' || !await hasEntitlement('permission-syncing')) { - res.status(403).json({ error: 'Permission syncing is not enabled.' }); - return; - } - - const schema = z.object({ - accountId: z.string(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const { accountId } = parsed.data; - const account = await this.prisma.account.findUnique({ - where: { id: accountId }, - }); - - if (!account) { - res.status(404).json({ error: 'Account not found' }); - return; - } - - if (!doesIdpSupportPermissionSyncing(account.providerType)) { - res.status(400).json({ error: `Provider '${account.providerType}' does not support permission syncing.` }); - return; - } - - const jobId = await this.accountPermissionSyncer.schedulePermissionSyncForAccount(account); - res.status(200).json({ jobId }); - } - - private async experimental_addGithubRepo(req: Request, res: Response) { - const schema = z.object({ - owner: z.string(), - repo: z.string(), - }).strict(); - - const parsed = schema.safeParse(req.body); - if (!parsed.success) { - res.status(400).json({ error: parsed.error.message }); - return; - } - - const octokit = new Octokit(); - let response; - try { - response = await octokit.rest.repos.get({ - owner: parsed.data.owner, - repo: parsed.data.repo, - }); - } catch (error) { - if (isNotFound(error)) { - res.status(404).json({ error: 'Repository not found on GitHub' }); - return; - } - throw error; - } - - const record = createGitHubRepoRecord({ - repo: response.data, - hostUrl: 'https://github.com', - isAutoCleanupDisabled: true, - }); - - const repo = await this.prisma.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: record.external_id, - external_codeHostUrl: record.external_codeHostUrl, - orgId: SINGLE_TENANT_ORG_ID, - } - }, - update: record, - create: record, - }); - - const [jobId ] = await this.repoIndexManager.createJobs([repo], RepoIndexingJobType.INDEX); - - res.status(200).json({ jobId, repoId: repo.id }); - } - public async dispose() { return new Promise((resolve, reject) => { this.server.close((err) => { diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index 9051164a7..84f85d023 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -755,7 +755,7 @@ export const getReposForAuthenticatedBitbucketServerUser = async ( * @note This only covers direct user-to-repo grants. It does NOT include users who have access via: * - Project-level permissions (inherited by all repos in the project) * - Group membership - * These users will still gain access through account-driven syncing (accountPermissionSyncer). + * These users will still gain access through account-driven permission syncing. * * @see https://developer.atlassian.com/server/bitbucket/rest/v906/api-group-repository/#api-rest-api-latest-projects-projectkey-repos-reposlug-permissions-users-get */ diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index 4d4f61ff6..8d04778a5 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -1,12 +1,13 @@ -import { Prisma, PrismaClient } from "@sourcebot/db"; +import { Prisma } from "@sourcebot/db"; import { createLogger, env } from "@sourcebot/shared"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { loadConfig } from "@sourcebot/shared"; import chokidar, { FSWatcher } from 'chokidar'; -import { ConnectionManager } from "./connectionManager.js"; import { SINGLE_TENANT_ORG_ID } from "./constants.js"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import isEqual from 'fast-deep-equal'; +import { JobManager } from "./types.js"; +import { prisma } from "./prisma.js"; const logger = createLogger('config-manager'); @@ -14,8 +15,7 @@ export class ConfigManager { private watcher: FSWatcher; constructor( - private db: PrismaClient, - private connectionManager: ConnectionManager, + private jobManager: JobManager, configPath: string, ) { this.watcher = chokidar.watch(configPath, { @@ -46,14 +46,13 @@ export class ConfigManager { await syncSearchContexts({ contexts: config.contexts, orgId: SINGLE_TENANT_ORG_ID, - db: this.db, }); } private syncConnections = async (connections?: { [key: string]: ConnectionConfig }) => { if (connections) { for (const [key, newConnectionConfig] of Object.entries(connections)) { - const existingConnection = await this.db.connection.findUnique({ + const existingConnection = await prisma.connection.findUnique({ where: { name_orgId: { name: key, @@ -73,7 +72,7 @@ export class ConfigManager { // Either update the existing connection or create a new one. const connection = existingConnection ? - await this.db.connection.update({ + await prisma.connection.update({ where: { id: existingConnection.id, }, @@ -84,7 +83,7 @@ export class ConfigManager { enforcePermissionsForPublicRepos, } }) : - await this.db.connection.create({ + await prisma.connection.create({ data: { name: key, config: newConnectionConfig as unknown as Prisma.InputJsonValue, @@ -102,13 +101,16 @@ export class ConfigManager { if (connectionNeedsSyncing) { logger.debug(`Change detected for connection '${key}' (id: ${connection.id}). Creating sync job.`); - await this.connectionManager.createJobs([connection]); + await this.jobManager.trigger('connection-sync', { + connectionId: connection.id, + orgId: SINGLE_TENANT_ORG_ID, + }) } } } // Delete any connections that are no longer in the config. - const deletedConnections = await this.db.connection.findMany({ + const deletedConnections = await prisma.connection.findMany({ where: { isDeclarative: true, name: { @@ -120,7 +122,7 @@ export class ConfigManager { for (const connection of deletedConnections) { logger.debug(`Deleting connection with name '${connection.name}'. Connection ID: ${connection.id}`); - await this.db.connection.delete({ + await prisma.connection.delete({ where: { id: connection.id, } @@ -131,4 +133,4 @@ export class ConfigManager { public dispose = async () => { await this.watcher.close(); } -} \ No newline at end of file +} diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts deleted file mode 100644 index c9db057e2..000000000 --- a/packages/backend/src/connectionManager.ts +++ /dev/null @@ -1,410 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { Connection, ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { createLogger, env, loadConfig } from "@sourcebot/shared"; -import { Job, Queue, Worker } from "bullmq"; -import { Redis } from 'ioredis'; -import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; -import { syncSearchContexts } from "./ee/syncSearchContexts.js"; -import { captureEvent } from "./posthog.js"; -import { PromClient } from "./promClient.js"; -import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; -import { Settings } from "./types.js"; -import { setIntervalAsync } from "./utils.js"; - -const LOG_TAG = 'connection-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'connection-sync-queue'; - -const CONNECTION_SYNC_TIMEOUT_MS = 1000 * 60 * 60 * 2; // 2 hours - -type JobPayload = { - jobId: string, - connectionId: number, - connectionName: string, - orgId: number, -}; - -type JobResult = { - repoCount: number, -} - -export class ConnectionManager { - private worker: Worker; - private queue: Queue; - private abortController: AbortController; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.worker = new Worker( - QUEUE_NAME, - this.runJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxConnectionSyncJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Connection syncer worker error:`, error); - }); - } - - public startScheduler() { - logger.debug('Starting scheduler'); - this.interval = setIntervalAsync(async () => { - const thresholdDate = new Date(Date.now() - this.settings.resyncConnectionIntervalMs); - const timeoutDate = new Date(Date.now() - CONNECTION_SYNC_TIMEOUT_MS); - - const connections = await this.db.connection.findMany({ - where: { - AND: [ - { - OR: [ - { syncedAt: null }, - { syncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - syncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { status: { in: [ConnectionSyncJobStatus.PENDING, ConnectionSyncJobStatus.IN_PROGRESS] } }, - { createdAt: { gt: timeoutDate } }, - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: ConnectionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - } - ] - } - }); - - if (connections.length > 0) { - await this.createJobs(connections); - } - }, this.settings.resyncConnectionPollingIntervalMs); - } - - - public async createJobs(connections: Connection[]) { - const jobs = await this.db.connectionSyncJob.createManyAndReturn({ - data: connections.map(connection => ({ - connectionId: connection.id, - })), - include: { - connection: true, - } - }); - - for (const job of jobs) { - logger.debug(`Scheduling job ${job.id} for connection ${job.connection.name} (id: ${job.connectionId})`); - await this.queue.add( - 'connection-sync-job', - { - jobId: job.id, - connectionId: job.connectionId, - connectionName: job.connection.name, - orgId: job.connection.orgId, - }, - { jobId: job.id } - ); - - this.promClient.pendingConnectionSyncJobs.inc({ connection: job.connection.name }); - } - - return jobs.map(job => job.id); - } - - private async runJob(job: Job): Promise { - const { jobId, connectionName } = job.data; - const logger = createJobLogger(jobId); - logger.debug(`Running connection sync job ${jobId} for connection ${connectionName} (id: ${job.data.connectionId})`); - - const currentStatus = await this.db.connectionSyncJob.findUniqueOrThrow({ - where: { - id: jobId, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if (currentStatus.status !== ConnectionSyncJobStatus.PENDING && currentStatus.status !== ConnectionSyncJobStatus.IN_PROGRESS) { - throw new Error(`Job ${jobId} is not in a valid state. Expected: ${ConnectionSyncJobStatus.PENDING} or ${ConnectionSyncJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - this.promClient.pendingConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.activeConnectionSyncJobs.inc({ connection: connectionName }); - - const { connection: { config: rawConnectionConfig, orgId } } = await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - }, - select: { - connection: { - select: { - config: true, - orgId: true, - } - } - }, - }); - - const config = rawConnectionConfig as unknown as ConnectionConfig; - - const result = await (async () => { - switch (config.type) { - case 'github': { - return await compileGithubConfig(config, job.data.connectionId, this.abortController.signal); - } - case 'gitlab': { - return await compileGitlabConfig(config, job.data.connectionId); - } - case 'gitea': { - return await compileGiteaConfig(config, job.data.connectionId); - } - case 'gerrit': { - return await compileGerritConfig(config, job.data.connectionId); - } - case 'bitbucket': { - return await compileBitbucketConfig(config, job.data.connectionId); - } - case 'azuredevops': { - return await compileAzureDevOpsConfig(config, job.data.connectionId); - } - case 'git': { - return await compileGenericGitHostConfig(config, job.data.connectionId); - } - } - })(); - - let { repoData, warnings } = result; - - await this.db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, - }); - - - // Filter out any duplicates by external_id and external_codeHostUrl. - repoData = repoData.filter((repo, index, self) => { - return index === self.findIndex(r => - r.external_id === repo.external_id && - r.external_codeHostUrl === repo.external_codeHostUrl - ); - }) - - // @note: to handle orphaned Repos we delete all RepoToConnection records for this connection, - // and then recreate them when we upsert the repos. For example, if a repo is no-longer - // captured by the connection's config (e.g., it was deleted, marked archived, etc.), it won't - // appear in the repoData array above, and so the RepoToConnection record won't be re-created. - // Repos that have no RepoToConnection records are considered orphaned and can be deleted. - await this.db.$transaction(async (tx) => { - const deleteStart = performance.now(); - await tx.connection.update({ - where: { - id: job.data.connectionId, - }, - data: { - repos: { - deleteMany: {} - } - } - }); - const deleteDuration = performance.now() - deleteStart; - logger.debug(`Deleted all RepoToConnection records for connection ${connectionName} (id: ${job.data.connectionId}) in ${deleteDuration}ms`); - - const totalUpsertStart = performance.now(); - for (const repo of repoData) { - const upsertStart = performance.now(); - await tx.repo.upsert({ - where: { - external_id_external_codeHostUrl_orgId: { - external_id: repo.external_id, - external_codeHostUrl: repo.external_codeHostUrl, - orgId: orgId, - } - }, - update: repo, - create: repo, - }) - const upsertDuration = performance.now() - upsertStart; - logger.debug(`Upserted repo ${repo.displayName} (id: ${repo.external_id}) in ${upsertDuration}ms`); - } - const totalUpsertDuration = performance.now() - totalUpsertStart; - logger.debug(`Upserted ${repoData.length} repos for connection ${connectionName} (id: ${job.data.connectionId}) in ${totalUpsertDuration}ms`); - }, { timeout: env.CONNECTION_MANAGER_UPSERT_TIMEOUT_MS }); - - return { - repoCount: repoData.length, - }; - } - - - private async onJobCompleted(job: Job, result: JobResult) { - try { - const logger = createJobLogger(job.id!); - const { connectionId, connectionName, orgId } = job.data; - - const { connection } = await this.db.connectionSyncJob.update({ - where: { - id: job.id!, - }, - data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - connection: { - update: { - syncedAt: new Date(), - } - } - }, - select: { - connection: true, - } - }); - - // After a connection has synced, we need to re-sync the org's search contexts as - // there may be new repos that match the search context's include/exclude patterns. - if (env.CONFIG_PATH) { - try { - const config = await loadConfig(env.CONFIG_PATH); - - await syncSearchContexts({ - db: this.db, - orgId, - contexts: config.contexts, - }); - } catch (err) { - logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`); - Sentry.captureException(err); - } - } - - logger.debug(`Connection sync job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) completed`); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connectionName }); - this.promClient.connectionSyncJobSuccessTotal.inc({ connection: connectionName }); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_completed', { - connectionId: connectionId, - repoCount: result.repoCount, - type: config.type, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - const jobLogger = createJobLogger(job.id!); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for connection ${job.data.connectionName} (id: ${job.data.connectionId}) failed. Retrying...`); - return; - } - - const { connection } = await this.db.connectionSyncJob.update({ - where: { id: job.id }, - data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: job.failedReason, - }, - select: { - connection: true, - } - }); - - this.promClient.activeConnectionSyncJobs.dec({ connection: connection.name }); - this.promClient.connectionSyncJobFailTotal.inc({ connection: connection.name }); - - jobLogger.error(`Failed job ${job.id} for connection ${connection.name} (id: ${connection.id}). Reason: ${job.failedReason}`); - - const config = connection.config as unknown as ConnectionConfig; - captureEvent('backend_connection_sync_job_failed', { - connectionId: job.data.connectionId, - type: config.type, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - await this.queue.close(); - } -} diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionWorkload.test.ts new file mode 100644 index 000000000..d55deb752 --- /dev/null +++ b/packages/backend/src/connectionWorkload.test.ts @@ -0,0 +1,426 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { PrismaClient } from "@sourcebot/db"; + +const mocks = vi.hoisted(() => ({ + connectionFindUniqueOrThrow: vi.fn(), + connectionUpdate: vi.fn(), + connectionSyncJobUpsert: vi.fn(), + connectionSyncJobUpdate: vi.fn(), + repoFindMany: vi.fn(), + repoUpsert: vi.fn(), + repoToConnectionDeleteMany: vi.fn(), + compileGithubConfig: vi.fn(), + loadConfig: vi.fn(), + syncSearchContexts: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@sourcebot/shared", () => ({ + CONNECTION_QUEUE: { + name: "connection-sync", + dedupKey: ({ connectionId }: { connectionId: number }) => + `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, + createLogger: vi.fn(() => ({ + debug: vi.fn(), + error: vi.fn(), + })), + env: { + CONFIG_PATH: "/config.json", + CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: 60_000, + }, + loadConfig: mocks.loadConfig, +})); + +vi.mock("./repoCompileUtils.js", () => ({ + compileAzureDevOpsConfig: vi.fn(), + compileBitbucketConfig: vi.fn(), + compileGenericGitHostConfig: vi.fn(), + compileGerritConfig: vi.fn(), + compileGiteaConfig: vi.fn(), + compileGithubConfig: mocks.compileGithubConfig, + compileGitlabConfig: vi.fn(), +})); + +vi.mock("./ee/syncSearchContexts.js", () => ({ + syncSearchContexts: mocks.syncSearchContexts, +})); + +import { createConnectionWorkload } from "./connectionWorkload.js"; + +const db = { + connection: { + findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, + update: mocks.connectionUpdate, + }, + repo: { + findMany: mocks.repoFindMany, + upsert: mocks.repoUpsert, + }, + repoToConnection: { + deleteMany: mocks.repoToConnectionDeleteMany, + }, + connectionSyncJob: { + upsert: mocks.connectionSyncJobUpsert, + update: mocks.connectionSyncJobUpdate, + }, +} as unknown as PrismaClient; + +const connectionWorkload = createConnectionWorkload({ + db, + settings: { + maxConnectionSyncJobConcurrency: 2, + } as never, +}); + +const data = { + connectionId: 42, + orgId: 7, +}; + +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const lifecycleContext = { + data, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: lifecycleLogger, +}; + +describe("connectionWorkload", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test("declares database-backed lifecycle hooks", () => { + expect(connectionWorkload.onStarted).toBeTypeOf("function"); + expect(connectionWorkload.onCompleted).toBeTypeOf("function"); + expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("marks the connection sync job as in progress when started", async () => { + await connectionWorkload.onStarted?.(lifecycleContext); + + expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: "job-1", + connectionId: 42, + status: "IN_PROGRESS", + warningMessages: [], + }, + }); + }); + + test("marks the connection sync job as completed", async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, { + reposToCleanup: [], + reposToIndex: [], + }); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + }); + + test("marks the connection sync job as failed after terminal failure", async () => { + await connectionWorkload.onTerminalFailure?.( + lifecycleContext, + new Error("Connection credentials expired"), + ); + + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Connection credentials expired", + }, + }); + }); + + test("discovers repositories using the connection provider", async () => { + const config = { + type: "github" as const, + }; + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [], + warnings: ["Repository was archived"], + }); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.repoFindMany.mockResolvedValue([]); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + const updateProgress = vi.fn(); + const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), + }; + const signal = new AbortController().signal; + + const result = await connectionWorkload.process({ + ...lifecycleContext, + signal, + logger, + updateProgress, + trigger: vi.fn(), + }); + + expect(mocks.compileGithubConfig).toHaveBeenCalledWith( + config, + 42, + signal, + ); + expect(logger.info).toHaveBeenCalledWith("Discovered 0 repositories", { + connectionId: 42, + repositoryCount: 0, + }); + expect(updateProgress).not.toHaveBeenCalled(); + expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + warningMessages: ["Repository was archived"], + }, + }); + expect(result).toEqual({ + reposToCleanup: [], + reposToIndex: [], + }); + }); + + test("finds orphaned repositories and repositories needing a first index", async () => { + const config = { + type: "github" as const, + }; + const existingIndexedAt = new Date("2026-07-30T12:00:00.000Z"); + const existingRepo = { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-1", + connections: { + create: { + connectionId: 42, + }, + }, + }; + const newRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-4", + connections: { + create: { + connectionId: 42, + }, + }, + }; + + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [existingRepo, newRepo], + warnings: [], + }); + mocks.repoFindMany + .mockResolvedValueOnce([{ id: 1 }, { id: 2 }, { id: 3 }]) + .mockResolvedValueOnce([{ id: 2, name: "github.com/sourcebot/repo-2" }]); + mocks.repoUpsert + .mockResolvedValueOnce({ + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt: existingIndexedAt, + }) + .mockResolvedValueOnce({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + mocks.repoToConnectionDeleteMany.mockResolvedValue({ count: 2 }); + mocks.connectionUpdate.mockResolvedValue({}); + mocks.loadConfig.mockResolvedValue({ contexts: undefined }); + mocks.syncSearchContexts.mockResolvedValue(undefined); + const trigger = vi.fn(); + + const result = await connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + }); + + expect(mocks.repoFindMany).toHaveBeenNthCalledWith(1, { + where: { + connections: { + some: { + connectionId: 42, + }, + }, + }, + select: { + id: true, + }, + }); + expect(mocks.repoUpsert).toHaveBeenNthCalledWith(1, { + where: { + external_id_external_codeHostUrl_orgId: { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + orgId: 7, + }, + }, + update: { + ...existingRepo, + connections: { + createMany: { + data: { + connectionId: 42, + }, + skipDuplicates: true, + }, + }, + }, + create: existingRepo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }); + expect(mocks.repoToConnectionDeleteMany).toHaveBeenCalledWith({ + where: { + connectionId: 42, + repoId: { + in: [2, 3], + }, + }, + }); + expect(mocks.repoFindMany).toHaveBeenNthCalledWith(2, { + where: { + id: { + in: [2, 3], + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }); + expect(result).toEqual({ + reposToCleanup: [ + { id: 2, name: "github.com/sourcebot/repo-2" }, + ], + reposToIndex: [ + { id: 4, name: "github.com/sourcebot/repo-4" }, + ], + }); + expect(trigger).toHaveBeenNthCalledWith(1, "repo-index", { + repoId: 2, + type: "CLEANUP", + }); + expect(trigger).toHaveBeenNthCalledWith(2, "repo-index", { + repoId: 4, + type: "INDEX", + }); + expect(mocks.repoUpsert.mock.invocationCallOrder[1]).toBeLessThan( + mocks.repoToConnectionDeleteMany.mock.invocationCallOrder[0], + ); + expect(mocks.repoFindMany.mock.invocationCallOrder[1]).toBeLessThan( + trigger.mock.invocationCallOrder[0], + ); + }); + + test("does not mark the connection synced when scheduling fails", async () => { + const config = { + type: "github" as const, + }; + const newRepo = { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + displayName: "sourcebot/repo-4", + connections: { + create: { + connectionId: 42, + }, + }, + }; + + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + config, + }); + mocks.compileGithubConfig.mockResolvedValue({ + repoData: [newRepo], + warnings: [], + }); + mocks.repoFindMany.mockResolvedValueOnce([]); + mocks.repoUpsert.mockResolvedValueOnce({ + id: 4, + name: "github.com/sourcebot/repo-4", + indexedAt: null, + }); + const trigger = vi.fn().mockRejectedValue(new Error("Redis unavailable")); + + await expect(connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + })).rejects.toThrow("Redis unavailable"); + + expect(trigger).toHaveBeenCalledWith("repo-index", { + repoId: 4, + type: "INDEX", + }); + expect(mocks.connectionUpdate).not.toHaveBeenCalled(); + expect(mocks.syncSearchContexts).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionWorkload.ts new file mode 100644 index 000000000..b6e429b50 --- /dev/null +++ b/packages/backend/src/connectionWorkload.ts @@ -0,0 +1,288 @@ +import { Settings, Workload } from "./types.js"; +import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; +import { compileAzureDevOpsConfig, compileBitbucketConfig, compileGenericGitHostConfig, compileGerritConfig, compileGiteaConfig, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js"; +import { CONNECTION_QUEUE, env, loadConfig } from "@sourcebot/shared"; +import { syncSearchContexts } from "./ee/syncSearchContexts.js"; +import * as Sentry from "@sentry/node"; +import { ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; + +interface Props { + db: PrismaClient, + settings: Settings; +} + +interface ConnectionSyncResult { + reposToCleanup: { id: number; name: string }[]; + reposToIndex: { id: number; name: string }[]; +} + +export const createConnectionWorkload = ({ + db, + settings +}: Props): Workload<'connection-sync', ConnectionSyncResult> => ({ + queueSpec: CONNECTION_QUEUE, + concurrency: settings.maxConnectionSyncJobConcurrency, + process: async ({ + data: { + connectionId, + orgId + }, + logger, + signal, + jobId, + trigger, + }) => { + logger.info(`Syncing connection ${connectionId}`, { + connectionId, + orgId, + }); + const connection = await db.connection.findUniqueOrThrow({ + where: { + id: connectionId + } + }); + + const config = connection.config as unknown as ConnectionConfig; + + const result = await discoverConnectionRepositories({ + config, + connectionId, + signal, + }); + + let { repoData, warnings } = result; + + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + warningMessages: warnings, + }, + }); + + logger.info(`Discovered ${repoData.length} repositories`, { + connectionId, + repositoryCount: repoData.length, + }); + + // Filter out any duplicates by external_id and external_codeHostUrl. + repoData = repoData.filter((repo, index, self) => { + return index === self.findIndex(r => + r.external_id === repo.external_id && + r.external_codeHostUrl === repo.external_codeHostUrl + ); + }) + + const previouslyAssociatedRepos = await db.repo.findMany({ + where: { + connections: { + some: { + connectionId, + }, + }, + }, + select: { + id: true, + }, + }); + + const upsertedRepos: { id: number; name: string; indexedAt: Date | null }[] = []; + + for (const repo of repoData) { + const upsertedRepo = await db.repo.upsert({ + where: { + external_id_external_codeHostUrl_orgId: { + external_id: repo.external_id, + external_codeHostUrl: repo.external_codeHostUrl, + orgId: orgId, + } + }, + update: { + ...repo, + connections: { + createMany: { + data: { + connectionId, + }, + skipDuplicates: true, + }, + }, + }, + create: repo, + select: { + id: true, + name: true, + indexedAt: true, + }, + }) + upsertedRepos.push(upsertedRepo); + } + + const currentRepoIds = new Set(upsertedRepos.map(({ id }) => id)); + const staleRepoIds = previouslyAssociatedRepos + .map(({ id }) => id) + .filter((id) => !currentRepoIds.has(id)); + + if (staleRepoIds.length > 0) { + await db.repoToConnection.deleteMany({ + where: { + connectionId, + repoId: { + in: staleRepoIds, + }, + }, + }); + } + + const reposToCleanup = staleRepoIds.length > 0 + ? await db.repo.findMany({ + where: { + id: { + in: staleRepoIds, + }, + connections: { + none: {}, + }, + }, + select: { + id: true, + name: true, + }, + }) + : []; + + const reposToIndex = upsertedRepos + .filter(({ indexedAt }) => indexedAt === null) + .map(({ id, name }) => ({ id, name })); + + await Promise.all(reposToCleanup.map(({ id }) => + trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }) + )); + + await Promise.all(reposToIndex.map(({ id }) => + trigger('repo-index', { + repoId: id, + type: 'INDEX', + }) + )); + + logger.info(`Stored ${repoData.length} repositories`, { + connectionId, + connectionName: connection.name, + repositoryCount: repoData.length, + }); + + await db.connection.update({ + where: { + id: connectionId, + }, + data: { + syncedAt: new Date(), + } + }); + + // After a connection has synced, we need to re-sync the org's search contexts as + // there may be new repos that match the search context's include/exclude patterns. + try { + const config = await loadConfig(env.CONFIG_PATH); + + await syncSearchContexts({ + orgId, + contexts: config.contexts, + }); + } catch (err) { + logger.error(`Failed to sync search contexts for connection ${connectionId}`, err); + Sentry.captureException(err); + } + + logger.info(`Connection ${connectionId} sync finished`, { + connectionId, + }); + + return { + reposToCleanup, + reposToIndex, + }; + }, + onStarted: async ({ data: { connectionId }, jobId }) => { + await db.connectionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: ConnectionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + warningMessages: [], + }, + create: { + id: jobId, + connectionId, + status: ConnectionSyncJobStatus.IN_PROGRESS, + warningMessages: [], + }, + }); + }, + onCompleted: async ({ jobId }) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + }, + onTerminalFailure: async ({ jobId }, error) => { + await db.connectionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: ConnectionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + }, +}); + +const discoverConnectionRepositories = async ({ + config, + connectionId, + signal, +}: { + config: ConnectionConfig; + connectionId: number; + signal: AbortSignal; +}) => { + switch (config.type) { + case 'github': { + return compileGithubConfig(config, connectionId, signal); + } + case 'gitlab': { + return compileGitlabConfig(config, connectionId); + } + case 'gitea': { + return compileGiteaConfig(config, connectionId); + } + case 'gerrit': { + return compileGerritConfig(config, connectionId); + } + case 'bitbucket': { + return compileBitbucketConfig(config, connectionId); + } + case 'azuredevops': { + return compileAzureDevOpsConfig(config, connectionId); + } + case 'git': { + return compileGenericGitHostConfig(config, connectionId); + } + } +}; diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..7326524a6 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -0,0 +1,423 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + ensureFreshAccountToken: vi.fn(), + getIdentityProviderConfig: vi.fn(), + getReposForAuthenticatedBitbucketCloudUser: vi.fn(), + getReposForAuthenticatedBitbucketServerUser: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + ACCOUNT_PERMISSION_SYNC_QUEUE: { + name: "account-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, + getIdentityProviderConfig: mocks.getIdentityProviderConfig, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser: + mocks.getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser: + mocks.getReposForAuthenticatedBitbucketServerUser, +})); + +vi.mock("./tokenRefresh.js", async (importOriginal) => ({ + ...(await importOriginal()), + ensureFreshAccountToken: mocks.ensureFreshAccountToken, +})); + +import { + classifyPermissionSyncFailure, + createAccountPermissionSyncWorkload, +} from "./accountPermissionSyncWorkload.js"; +import { + PermissionSyncUpstreamError, + type PermissionSyncUpstreamErrorKind, +} from "./permissionSyncError.js"; +import { + TokenRefreshError, + type TokenRefreshErrorKind, +} from "./tokenRefresh.js"; + +const tokenRefreshError = ( + kind: TokenRefreshErrorKind, + status?: number, +): TokenRefreshError => + new TokenRefreshError(`Token refresh failed: ${kind}`, { + kind, + status, + }); + +const upstreamError = ( + kind: PermissionSyncUpstreamErrorKind, +): PermissionSyncUpstreamError => + new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { + kind, + provider: "github", + operation: "list_accessible_repositories", + }); + +const account = { + id: "account_1", + providerId: "bitbucket-server", + issuerUrl: "https://bitbucket.example.com", + user: { email: "user@example.com" }, +}; +const accountFindUniqueOrThrow = vi.fn().mockResolvedValue(account); +const accountUpdate = vi.fn().mockResolvedValue(account); +const repoFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionDeleteMany = vi.fn().mockResolvedValue({ count: 95 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ account }); +const transaction = vi.fn((queries: Array>) => + Promise.all(queries), +); + +const db = { + account: { + findUniqueOrThrow: accountFindUniqueOrThrow, + update: accountUpdate, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + deleteMany: permissionDeleteMany, + }, + repo: { + findMany: repoFindMany, + }, + accountPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createAccountPermissionSyncWorkload({ + db, + settings: { + maxAccountPermissionSyncJobConcurrency: 2, + } as never, + }); + +const lifecycleContext = { + data: { + accountId: "account_1", + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.ensureFreshAccountToken.mockReset().mockResolvedValue("access-token"); + mocks.getIdentityProviderConfig.mockReset().mockResolvedValue({ + provider: "bitbucket-server", + baseUrl: "https://bitbucket.example.com", + }); + mocks.createBitbucketServerClient.mockReset().mockReturnValue({}); + mocks.getReposForAuthenticatedBitbucketServerUser + .mockReset() + .mockResolvedValue([]); + accountFindUniqueOrThrow.mockResolvedValue(account); + repoFindMany.mockResolvedValue([]); + permissionCreateMany.mockResolvedValue({ count: 0 }); + permissionDeleteMany.mockResolvedValue({ count: 95 }); + permissionSyncJobUpdate.mockResolvedValue({ account }); +}); + +describe("classifyPermissionSyncFailure", () => { + test("fails closed when the refresh token is rejected", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("refresh_token_rejected", 400), + ), + ).toEqual({ + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + }); + }); + + test.each([ + ["transient", 500], + ["configuration", 400], + ["invalid_response", undefined], + ["local_credential", undefined], + ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)( + "keeps permissions for a %s token refresh failure", + (kind, status) => { + expect( + classifyPermissionSyncFailure(tokenRefreshError(kind, status)), + ).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test("does not treat a token refresh configuration error with HTTP 401 as an API authorization failure", () => { + expect( + classifyPermissionSyncFailure( + tokenRefreshError("configuration", 401), + ), + ).toEqual({ + action: "preserve_permissions", + }); + }); + + test.each([ + ["credential_rejected", "upstream_credential_rejected"], + ["insufficient_scope", "upstream_insufficient_scope"], + ] as const)( + "fails closed for a classified %s upstream failure", + (kind, reason) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "clear_permissions", + reason, + }); + }, + ); + + test.each([ + "rate_limited", + "upstream_unavailable", + "forbidden", + "unknown", + ] satisfies PermissionSyncUpstreamErrorKind[])( + "keeps permissions for a classified %s upstream failure", + (kind) => { + expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ + action: "preserve_permissions", + }); + }, + ); + + test.each([401, 403, 410])( + "does not fail closed on an unclassified HTTP %s error", + (status) => { + const error = Object.assign(new Error(`HTTP ${status}`), { + status, + }); + expect(classifyPermissionSyncFailure(error)).toEqual({ + action: "preserve_permissions", + }); + }, + ); +}); + +describe("accountPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("account-permission-sync"); + expect(workload.concurrency).toBe(2); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("syncs the requested account", async () => { + const workload = createWorkload(); + + await workload.process(processContext); + + expect(accountFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: "account_1" }, + include: { user: true }, + }); + expect(mocks.ensureFreshAccountToken).toHaveBeenCalledWith(account, db); + expect(mocks.createBitbucketServerClient).toHaveBeenCalledWith( + "https://bitbucket.example.com", + undefined, + "access-token", + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(accountFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("atomically records a reauthentication issue when the refresh token is rejected", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).toHaveBeenCalledWith({ + where: { accountId: "account_1" }, + }); + expect(accountUpdate).toHaveBeenCalledWith({ + where: { id: "account_1" }, + data: { + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + permissionSyncIssueAt: expect.any(Date), + }, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an issue even when permissions were cleared by an earlier attempt", async () => { + const error = tokenRefreshError("refresh_token_rejected", 400); + permissionDeleteMany.mockResolvedValue({ count: 0 }); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "REAUTHENTICATION_REQUIRED", + }), + }), + ); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("records an insufficient-scope issue for scope failures", async () => { + const error = upstreamError("insufficient_scope"); + mocks.getReposForAuthenticatedBitbucketServerUser.mockRejectedValue( + error, + ); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(accountUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + permissionSyncIssue: "INSUFFICIENT_SCOPE", + }), + }), + ); + }); + + test("preserves permissions for a transient refresh failure", async () => { + const error = tokenRefreshError("transient", 500); + mocks.ensureFreshAccountToken.mockRejectedValue(error); + const workload = createWorkload(); + + await expect(workload.process(processContext)).rejects.toBe(error); + + expect(permissionDeleteMany).not.toHaveBeenCalled(); + expect(accountUpdate).not.toHaveBeenCalled(); + expect(transaction).not.toHaveBeenCalled(); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + accountId: "account_1", + status: "IN_PROGRESS", + }, + }); + }); + + test("marks a job completed and clears the account issue", async () => { + await createWorkload().onCompleted?.(lifecycleContext, undefined); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + account: { + update: { + permissionSyncedAt: expect.any(Date), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }, + }, + select: { + account: { + include: { user: true }, + }, + }, + }); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + }), + ); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "account-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.ts new file mode 100644 index 000000000..af5a3b721 --- /dev/null +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.ts @@ -0,0 +1,525 @@ +import * as Sentry from "@sentry/node"; +import { + Account, + AccountPermissionSyncIssue, + AccountPermissionSyncJobStatus, + PermissionSyncSource, + PrismaClient, +} from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + getIdentityProviderConfig, + JobLogSink, +} from "@sourcebot/shared"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getReposForAuthenticatedBitbucketCloudUser, + getReposForAuthenticatedBitbucketServerUser, +} from "../bitbucket.js"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, + getReposForAuthenticatedUser, +} from "../github.js"; +import { + createGitLabFromOAuthToken, + getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, + getProjectsForAuthenticatedUser, +} from "../gitlab.js"; +import { + PermissionSyncUpstreamError, + withPermissionSyncUpstreamError, +} from "./permissionSyncError.js"; +import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; +import { Settings, Workload } from "../types.js"; +import { IdentityProviderConfig } from "@sourcebot/schemas/v3/index.type"; + +type AccountWithUser = Account & { user: { email: string | null } }; + +type SupportedProvider = + | "github" + | "gitlab" + | "bitbucket-cloud" + | "bitbucket-server"; +type ProviderConfig = Extract< + IdentityProviderConfig, + { provider: TProvider } +>; + +interface ProviderPermissionSyncProps { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: ProviderConfig; +} + +export type PermissionCleanupReason = + | "oauth_refresh_token_rejected" + | "upstream_credential_rejected" + | "upstream_insufficient_scope"; + +export type PermissionCleanupDecision = + | { + action: "clear_permissions"; + reason: PermissionCleanupReason; + } + | { + action: "preserve_permissions"; + }; + +export const classifyPermissionSyncFailure = ( + error: unknown, +): PermissionCleanupDecision => { + // Token refresh failures have their own classification. Do not fall through + // to the generic HTTP checks because another token endpoint failure may + // also carry a 401 or 403 status. + if (error instanceof TokenRefreshError) { + return error.kind === "refresh_token_rejected" + ? { + action: "clear_permissions", + reason: "oauth_refresh_token_rejected", + } + : { action: "preserve_permissions" }; + } + + if (error instanceof PermissionSyncUpstreamError) { + if (error.kind === "credential_rejected") { + return { + action: "clear_permissions", + reason: "upstream_credential_rejected", + }; + } + if (error.kind === "insufficient_scope") { + return { + action: "clear_permissions", + reason: "upstream_insufficient_scope", + }; + } + } + + return { action: "preserve_permissions" }; +}; + +const PERMISSION_CLEANUP_DETAILS: Record< + PermissionCleanupReason, + { + message: string; + issue: AccountPermissionSyncIssue; + } +> = { + oauth_refresh_token_rejected: { + message: "OAuth refresh token rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_credential_rejected: { + message: "upstream credential rejection", + issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, + }, + upstream_insufficient_scope: { + message: "insufficient OAuth scope", + issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, + }, +}; + +interface AccountPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createAccountPermissionSyncWorkload = ({ + db, + settings, +}: AccountPermissionSyncWorkloadDependencies): Workload<"account-permission-sync"> => { + return { + queueSpec: ACCOUNT_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxAccountPermissionSyncJobConcurrency, + process: async ({ data: { accountId }, logger: jobLogger }) => { + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + const account = await db.account.findUniqueOrThrow({ + where: { + id: accountId, + }, + include: { + user: true, + }, + }); + + jobLogger.debug( + `Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`, + ); + + try { + // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. + const accessToken = await ensureFreshAccountToken(account, db); + + const idpConfig = await getIdentityProviderConfig( + account.providerId, + ); + if (!idpConfig) { + throw new Error( + "Unable to find IDP config in config.json.", + ); + } + + const repoIds = await getAccessibleRepoIds({ + db, + account, + accessToken, + config: idpConfig, + }); + + await db.$transaction([ + db.account.update({ + where: { + id: account.id, + }, + data: { + accessibleRepos: { + deleteMany: {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: repoIds.map((repoId) => ({ + accountId: account.id, + repoId, + source: PermissionSyncSource.ACCOUNT_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + } catch (error) { + // Clear cached permissions only for classified permanent failures. + // Ambiguous HTTP errors and transient upstream failures preserve the + // last successful permission state. + const cleanupDecision = classifyPermissionSyncFailure(error); + + if (cleanupDecision.action === "clear_permissions") { + const details = + PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; + const [{ count }] = await db.$transaction([ + db.accountToRepoPermission.deleteMany({ + where: { accountId: account.id }, + }), + db.account.update({ + where: { id: account.id }, + data: { + permissionSyncIssue: details.issue, + permissionSyncIssueAt: new Date(), + }, + }), + ]); + const message = + error instanceof Error ? error.message : String(error); + jobLogger.warn( + `Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? "unknown"}) — fail-closed cleanup triggered by ${details.message}: ${message}`, + ); + } + throw error; + } + }, + onStarted: async ({ data: { accountId }, jobId }) => { + await db.accountPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + accountId, + status: AccountPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId, logger: jobLogger }) => { + const { account } = await db.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + account: { + update: { + permissionSyncedAt: new Date(), + permissionSyncIssue: null, + permissionSyncIssueAt: null, + }, + }, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + + jobLogger.debug( + `Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`, + ); + }, + onTerminalFailure: async ( + { data: { accountId }, jobId, logger: jobLogger }, + error, + ) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: ACCOUNT_PERMISSION_SYNC_QUEUE.name, + }, + }); + + const { account } = await db.accountPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: AccountPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + select: { + account: { + include: { + user: true, + }, + }, + }, + }); + + jobLogger.error( + `Account permission sync job failed for account (id: ${accountId}) for user ${account.user.email ?? "unknown user (email not found)"}: ${error.message}`, + ); + }, + }; +}; + +const getAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: { + db: PrismaClient; + account: AccountWithUser; + accessToken: string; + config: IdentityProviderConfig; +}): Promise => { + switch (config.provider) { + case "github": + return getGitHubAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "gitlab": + return getGitLabAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-cloud": + return getBitbucketCloudAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + case "bitbucket-server": + return getBitbucketServerAccessibleRepoIds({ + db, + account, + accessToken, + config, + }); + default: + throw new Error(`Unsupported provider type: ${config.provider}`); + } +}; + +const getGitHubAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"github">): Promise => { + const { octokit } = await createOctokitFromToken({ + token: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "github", + "inspect_token_scopes", + () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), + ); + + // Token supports scope introspection (classic PAT or OAuth app token). + if (scopes !== null && !scopes.includes("repo")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, + { + kind: "insufficient_scope", + provider: "github", + operation: "inspect_token_scopes", + }, + ); + } + + // Public repos do not need an explicit permission mapping. + const githubRepos = await withPermissionSyncUpstreamError( + "github", + "list_accessible_repositories", + () => getReposForAuthenticatedUser("private", octokit), + ); + const gitHubRepoIds = githubRepos.map((repo) => repo.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "github", + external_id: { + in: gitHubRepoIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getGitLabAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"gitlab">): Promise => { + const api = await createGitLabFromOAuthToken({ + oauthToken: accessToken, + url: config.baseUrl, + }); + + const scopes = await withPermissionSyncUpstreamError( + "gitlab", + "inspect_token_scopes", + () => getGitLabOAuthScopesForAuthenticatedUser(api), + ); + if (!scopes.includes("read_api")) { + throw new PermissionSyncUpstreamError( + `OAuth token with scopes [${scopes.join(", ")}] is missing the 'read_api' scope required for permission syncing.`, + { + kind: "insufficient_scope", + provider: "gitlab", + operation: "inspect_token_scopes", + }, + ); + } + + // Public and internal repos do not need an explicit permission mapping. + const gitLabProjectIds = ( + await withPermissionSyncUpstreamError( + "gitlab", + "list_accessible_repositories", + () => getProjectsForAuthenticatedUser("private", api), + ) + ).map((project) => project.id.toString()); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "gitlab", + external_id: { + in: gitLabProjectIds, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketCloudAccessibleRepoIds = async ({ + db, + account, + accessToken, +}: ProviderPermissionSyncProps<"bitbucket-cloud">): Promise => { + // Use a bearer token by omitting the user. + const client = createBitbucketCloudClient(undefined, accessToken); + const bitbucketRepos = await withPermissionSyncUpstreamError( + "bitbucket-cloud", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketCloudUser(client), + ); + const bitbucketRepoUuids = bitbucketRepos.map((repo) => repo.uuid); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketCloud", + external_id: { + in: bitbucketRepoUuids, + }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; + +const getBitbucketServerAccessibleRepoIds = async ({ + db, + account, + accessToken, + config, +}: ProviderPermissionSyncProps<"bitbucket-server">): Promise => { + const client = createBitbucketServerClient( + config.baseUrl, + undefined, + accessToken, + ); + const serverRepos = await withPermissionSyncUpstreamError( + "bitbucket-server", + "list_accessible_repositories", + () => getReposForAuthenticatedBitbucketServerUser(client), + ); + const serverRepoIds = serverRepos.map((repo) => repo.id); + + const repos = await db.repo.findMany({ + where: { + external_codeHostType: "bitbucketServer", + external_id: { in: serverRepoIds }, + ...(account.issuerUrl + ? { + external_codeHostUrl: account.issuerUrl, + } + : {}), + }, + }); + + return repos.map((repo) => repo.id); +}; diff --git a/packages/backend/src/ee/accountPermissionSyncer.test.ts b/packages/backend/src/ee/accountPermissionSyncer.test.ts deleted file mode 100644 index 09b35d594..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.test.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; - -const mocks = vi.hoisted(() => ({ - hasEntitlement: vi.fn(), -})); - -vi.mock('../entitlements.js', () => ({ - hasEntitlement: mocks.hasEntitlement, -})); - -import { AccountPermissionSyncer, classifyPermissionSyncFailure } from './accountPermissionSyncer.js'; -import { - PermissionSyncUpstreamError, - type PermissionSyncUpstreamErrorKind, -} from './permissionSyncError.js'; -import { TokenRefreshError, type TokenRefreshErrorKind } from './tokenRefresh.js'; - -const tokenRefreshError = ( - kind: TokenRefreshErrorKind, - status?: number, -): TokenRefreshError => new TokenRefreshError(`Token refresh failed: ${kind}`, { - kind, - status, -}); - -const upstreamError = ( - kind: PermissionSyncUpstreamErrorKind, -): PermissionSyncUpstreamError => new PermissionSyncUpstreamError(`Permission sync failed: ${kind}`, { - kind, - provider: 'github', - operation: 'list_accessible_repositories', -}); - -const createSyncerHarness = (syncError?: Error, permissionCount = 95) => { - const account = { - id: 'account_1', - providerId: 'bitbucket-server', - user: { email: 'user@example.com' }, - }; - const db = { - accountPermissionSyncJob: { - update: vi.fn().mockResolvedValue({ account }), - }, - accountToRepoPermission: { - deleteMany: vi.fn().mockResolvedValue({ count: permissionCount }), - }, - account: { - update: vi.fn().mockResolvedValue(account), - }, - $transaction: vi.fn((queries: Array>) => Promise.all(queries)), - }; - const syncAccountPermissions = syncError - ? vi.fn().mockRejectedValue(syncError) - : vi.fn().mockResolvedValue(undefined); - const syncer = Object.create(AccountPermissionSyncer.prototype) as { - db: typeof db; - syncAccountPermissions: typeof syncAccountPermissions; - runJob(job: { data: { jobId: string } }): Promise; - onJobCompleted(job: { data: { jobId: string } }): Promise; - }; - syncer.db = db; - syncer.syncAccountPermissions = syncAccountPermissions; - - return { - account, - db, - job: { data: { jobId: 'job_1' } }, - syncer, - }; -}; - -beforeEach(() => { - vi.clearAllMocks(); - mocks.hasEntitlement.mockResolvedValue(true); -}); - -describe('classifyPermissionSyncFailure', () => { - test('fails closed when the refresh token is rejected', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('refresh_token_rejected', 400))).toEqual({ - action: 'clear_permissions', - reason: 'oauth_refresh_token_rejected', - }); - }); - - test.each([ - ['transient', 500], - ['configuration', 400], - ['invalid_response', undefined], - ['local_credential', undefined], - ] satisfies Array<[TokenRefreshErrorKind, number | undefined]>)('keeps permissions for a %s token refresh failure', (kind, status) => { - expect(classifyPermissionSyncFailure(tokenRefreshError(kind, status))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test('does not treat a token refresh configuration error with HTTP 401 as an API authorization failure', () => { - expect(classifyPermissionSyncFailure(tokenRefreshError('configuration', 401))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([ - ['credential_rejected', 'upstream_credential_rejected'], - ['insufficient_scope', 'upstream_insufficient_scope'], - ] as const)('fails closed for a classified %s upstream failure', (kind, reason) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'clear_permissions', - reason, - }); - }); - - test.each([ - 'rate_limited', - 'upstream_unavailable', - 'forbidden', - 'unknown', - ] satisfies PermissionSyncUpstreamErrorKind[])('keeps permissions for a classified %s upstream failure', (kind) => { - expect(classifyPermissionSyncFailure(upstreamError(kind))).toEqual({ - action: 'preserve_permissions', - }); - }); - - test.each([401, 403, 410])('does not fail closed on an unclassified HTTP %s error', (status) => { - const error = Object.assign(new Error(`HTTP ${status}`), { status }); - expect(classifyPermissionSyncFailure(error)).toEqual({ - action: 'preserve_permissions', - }); - }); -}); - -describe('permission sync issue lifecycle', () => { - test('atomically records a reauthentication issue when the refresh token is rejected', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).toHaveBeenCalledWith({ - where: { accountId: 'account_1' }, - }); - expect(db.account.update).toHaveBeenCalledWith({ - where: { id: 'account_1' }, - data: { - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - permissionSyncIssueAt: expect.any(Date), - }, - }); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an issue when permissions were already cleared by an earlier attempt', async () => { - const error = tokenRefreshError('refresh_token_rejected', 400); - const { db, job, syncer } = createSyncerHarness(error, 0); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', - }), - })); - expect(db.$transaction).toHaveBeenCalledOnce(); - }); - - test('records an insufficient-scope issue for scope failures', async () => { - const error = upstreamError('insufficient_scope'); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.account.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - permissionSyncIssue: 'INSUFFICIENT_SCOPE', - }), - })); - }); - - test('does not record an issue or clear permissions for a transient refresh failure', async () => { - const error = tokenRefreshError('transient', 500); - const { db, job, syncer } = createSyncerHarness(error); - - await expect(syncer.runJob(job)).rejects.toBe(error); - - expect(db.accountToRepoPermission.deleteMany).not.toHaveBeenCalled(); - expect(db.account.update).not.toHaveBeenCalled(); - expect(db.$transaction).not.toHaveBeenCalled(); - }); - - test('clears the action-required issue after a successful permission sync', async () => { - const { db, job, syncer } = createSyncerHarness(); - - await syncer.onJobCompleted(job); - - expect(db.accountPermissionSyncJob.update).toHaveBeenCalledWith(expect.objectContaining({ - data: expect.objectContaining({ - account: { - update: expect.objectContaining({ - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }), - }, - }), - })); - }); -}); diff --git a/packages/backend/src/ee/accountPermissionSyncer.ts b/packages/backend/src/ee/accountPermissionSyncer.ts deleted file mode 100644 index 7184a191e..000000000 --- a/packages/backend/src/ee/accountPermissionSyncer.ts +++ /dev/null @@ -1,535 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { - PrismaClient, - AccountPermissionSyncIssue, - AccountPermissionSyncJobStatus, - Account, - PermissionSyncSource, -} from "@sourcebot/db"; -import { env, createLogger, getIdentityProviderConfig, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { ensureFreshAccountToken, TokenRefreshError } from "./tokenRefresh.js"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { Redis } from "ioredis"; -import { - createOctokitFromToken, - getOAuthScopesForAuthenticatedUser as getGitHubOAuthScopesForAuthenticatedUser, - getReposForAuthenticatedUser, -} from "../github.js"; -import { - createGitLabFromOAuthToken, - getOAuthScopesForAuthenticatedUser as getGitLabOAuthScopesForAuthenticatedUser, - getProjectsForAuthenticatedUser, -} from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getReposForAuthenticatedBitbucketCloudUser, getReposForAuthenticatedBitbucketServerUser } from "../bitbucket.js"; -import { Settings } from "../types.js"; -import { setIntervalAsync } from "../utils.js"; -import { PermissionSyncUpstreamError, withPermissionSyncUpstreamError } from "./permissionSyncError.js"; - -const LOG_TAG = 'user-permission-syncer'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -const QUEUE_NAME = 'accountPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; - -type AccountPermissionSyncJob = { - jobId: string; -} - -export type PermissionCleanupReason = - | 'oauth_refresh_token_rejected' - | 'upstream_credential_rejected' - | 'upstream_insufficient_scope'; - -export type PermissionCleanupDecision = - | { - action: 'clear_permissions'; - reason: PermissionCleanupReason; - } - | { - action: 'preserve_permissions'; - }; - -const PERMISSION_CLEANUP_DETAILS: Record = { - oauth_refresh_token_rejected: { - message: 'OAuth refresh token rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_credential_rejected: { - message: 'upstream credential rejection', - issue: AccountPermissionSyncIssue.REAUTHENTICATION_REQUIRED, - }, - upstream_insufficient_scope: { - message: 'insufficient OAuth scope', - issue: AccountPermissionSyncIssue.INSUFFICIENT_SCOPE, - }, -}; - -export const classifyPermissionSyncFailure = (error: unknown): PermissionCleanupDecision => { - // Token refresh failures have their own classification. Do not fall through - // to the generic HTTP checks because another token endpoint failure may - // also carry a 401 or 403 status. - if (error instanceof TokenRefreshError) { - return error.kind === 'refresh_token_rejected' - ? { action: 'clear_permissions', reason: 'oauth_refresh_token_rejected' } - : { action: 'preserve_permissions' }; - } - - if (error instanceof PermissionSyncUpstreamError) { - if (error.kind === 'credential_rejected') { - return { action: 'clear_permissions', reason: 'upstream_credential_rejected' }; - } - if (error.kind === 'insufficient_scope') { - return { action: 'clear_permissions', reason: 'upstream_insufficient_scope' }; - } - } - - return { action: 'preserve_permissions' }; -}; - -export class AccountPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxAccountPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - const thresholdDate = new Date(Date.now() - this.settings.userDrivenPermissionSyncIntervalMs); - - const accounts = await this.db.account.findMany({ - where: { - AND: [ - { - providerType: { - in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS - } - }, - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - AccountPermissionSyncJobStatus.PENDING, - AccountPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: AccountPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(accounts); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - public async schedulePermissionSyncForAccount(account: Account) { - const [job] = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: [{ accountId: account.id }], - }); - - await this.queue.add('accountPermissionSyncJob', { - jobId: job.id, - }, { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - priority: 1, - }); - - return job.id; - } - - private async schedulePermissionSync(accounts: Account[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.accountPermissionSyncJob.createManyAndReturn({ - data: accounts.map(account => ({ - accountId: account.id, - })), - include: { - account: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'accountPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.account.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - try { - await this.syncAccountPermissions(account, logger); - } catch (error) { - // Clear cached permissions only for classified permanent failures. - // Ambiguous HTTP errors and transient upstream failures preserve the - // last successful permission state. - const cleanupDecision = classifyPermissionSyncFailure(error); - - if (cleanupDecision.action === 'clear_permissions') { - const details = PERMISSION_CLEANUP_DETAILS[cleanupDecision.reason]; - const [{ count }] = await this.db.$transaction([ - this.db.accountToRepoPermission.deleteMany({ - where: { accountId: account.id }, - }), - this.db.account.update({ - where: { id: account.id }, - data: { - permissionSyncIssue: details.issue, - permissionSyncIssueAt: new Date(), - }, - }), - ]); - const message = error instanceof Error ? error.message : String(error); - logger.warn(`Cleared ${count} permission row(s) for account ${account.id} (user ${account.user.email ?? 'unknown'}) — fail-closed cleanup triggered by ${details.message}: ${message}`); - } - throw error; - } - } - - private async syncAccountPermissions( - account: Account & { user: { email: string | null } }, - logger: ReturnType, - ) { - logger.debug(`Syncing permissions for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}...`); - - // Ensure the OAuth token is fresh, refreshing it if it is expired or near expiry. - const accessToken = await ensureFreshAccountToken(account, this.db); - - // Get a list of all repos that the user has access to from all connected accounts. - const repoIds = await (async () => { - const aggregatedRepoIds: Set = new Set(); - - const idpConfig = await getIdentityProviderConfig(account.providerId); - - if (!idpConfig) { - throw new Error(`Unable to find IDP config in config.json.`); - } - - if (idpConfig.provider === 'github') { - const { octokit } = await createOctokitFromToken({ - token: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'github', - 'inspect_token_scopes', - () => getGitHubOAuthScopesForAuthenticatedUser(octokit, accessToken), - ); - - // Token supports scope introspection (classic PAT or OAuth app token) - if (scopes !== null) { - if (!scopes.includes('repo')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'repo' scope required for permission syncing. Please re-authorize with GitHub to grant the required scope.`, - { - kind: 'insufficient_scope', - provider: 'github', - operation: 'inspect_token_scopes', - }, - ); - } - } - - // @note: we only care about the private repos since we don't need to build a mapping - // for public repos. - // @see: packages/web/src/prisma.ts - const githubRepos = await withPermissionSyncUpstreamError( - 'github', - 'list_accessible_repositories', - () => getReposForAuthenticatedUser(/* visibility = */ 'private', octokit), - ); - const gitHubRepoIds = githubRepos.map(repo => repo.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'github', - external_id: { - in: gitHubRepoIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'gitlab') { - const api = await createGitLabFromOAuthToken({ - oauthToken: accessToken, - url: idpConfig.baseUrl, - }); - - const scopes = await withPermissionSyncUpstreamError( - 'gitlab', - 'inspect_token_scopes', - () => getGitLabOAuthScopesForAuthenticatedUser(api), - ); - if (!scopes.includes('read_api')) { - throw new PermissionSyncUpstreamError( - `OAuth token with scopes [${scopes.join(', ')}] is missing the 'read_api' scope required for permission syncing.`, - { - kind: 'insufficient_scope', - provider: 'gitlab', - operation: 'inspect_token_scopes', - }, - ); - } - - // @note: we only care about the private repos since we don't need to build a - // mapping for public or internal repos. Note that internal repos are _not_ - // enforced by permission syncing and therefore we don't need to fetch them - // here. - // - // @see: packages/web/src/prisma.ts - const gitLabProjectIds = ( - await withPermissionSyncUpstreamError( - 'gitlab', - 'list_accessible_repositories', - () => getProjectsForAuthenticatedUser('private', api), - ) - ).map(project => project.id.toString()); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'gitlab', - external_id: { - in: gitLabProjectIds, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-cloud') { - // @note: we don't pass a user here since we want to use a bearer token - // for authentication. - const client = createBitbucketCloudClient(/* user = */ undefined, accessToken) - const bitbucketRepos = await withPermissionSyncUpstreamError( - 'bitbucket-cloud', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketCloudUser(client), - ); - const bitbucketRepoUuids = bitbucketRepos.map(repo => repo.uuid); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketCloud', - external_id: { - in: bitbucketRepoUuids, - }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else if (idpConfig.provider === 'bitbucket-server') { - const client = createBitbucketServerClient(idpConfig.baseUrl, /* user = */ undefined, accessToken); - const serverRepos = await withPermissionSyncUpstreamError( - 'bitbucket-server', - 'list_accessible_repositories', - () => getReposForAuthenticatedBitbucketServerUser(client), - ); - const serverRepoIds = serverRepos.map(r => r.id); - - const repos = await this.db.repo.findMany({ - where: { - external_codeHostType: 'bitbucketServer', - external_id: { in: serverRepoIds }, - ...(account.issuerUrl ? { - external_codeHostUrl: account.issuerUrl, - } : {}), - } - }); - - repos.forEach(repo => aggregatedRepoIds.add(repo.id)); - } else { - throw new Error(`Unsupported provider type: ${idpConfig.provider}`); - } - - return Array.from(aggregatedRepoIds); - })(); - - await this.db.$transaction([ - this.db.account.update({ - where: { - id: account.id, - }, - data: { - accessibleRepos: { - deleteMany: {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: repoIds.map(repoId => ({ - accountId: account.id, - repoId, - source: PermissionSyncSource.ACCOUNT_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.COMPLETED, - account: { - update: { - permissionSyncedAt: new Date(), - permissionSyncIssue: null, - permissionSyncIssueAt: null, - }, - }, - completedAt: new Date(), - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.debug(`Permissions synced for ${account.providerId} account (id: ${account.id}) for user ${account.user.email}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (accountId: string, email: string) => `Account permission sync job failed for account (id: ${accountId}) for user ${email}: ${err.message}`; - - if (job) { - const { account } = await this.db.accountPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - account: { - include: { - user: true, - } - } - } - }); - - logger.error(errorMessage(account.id, account.user.email ?? 'unknown user (email not found)')); - } else { - logger.error(errorMessage('unknown account (id not found)', 'unknown user (id not found)')); - } - } -} diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts new file mode 100644 index 000000000..648c5b88d --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -0,0 +1,361 @@ +import type { PrismaClient } from "@sourcebot/db"; +import type { JobLogger } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureException: vi.fn(), + createBitbucketCloudClient: vi.fn(), + createBitbucketServerClient: vi.fn(), + createGitLabFromPersonalAccessToken: vi.fn(), + createOctokitFromToken: vi.fn(), + getAuthCredentialsForRepo: vi.fn(), + getExplicitUserPermissionsForCloudRepo: vi.fn(), + getProjectMembers: vi.fn(), + getRepoCollaborators: vi.fn(), + getUserPermissionsForServerRepo: vi.fn(), + hasEntitlement: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: mocks.captureException, +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + REPO_PERMISSION_SYNC_QUEUE: { + name: "repo-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, +})); + +vi.mock("../entitlements.js", () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +vi.mock("../utils.js", () => ({ + getAuthCredentialsForRepo: mocks.getAuthCredentialsForRepo, +})); + +vi.mock("../github.js", () => ({ + createOctokitFromToken: mocks.createOctokitFromToken, + getRepoCollaborators: mocks.getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME: "github.com", +})); + +vi.mock("../gitlab.js", () => ({ + createGitLabFromPersonalAccessToken: + mocks.createGitLabFromPersonalAccessToken, + getProjectMembers: mocks.getProjectMembers, +})); + +vi.mock("../bitbucket.js", () => ({ + createBitbucketCloudClient: mocks.createBitbucketCloudClient, + createBitbucketServerClient: mocks.createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo: + mocks.getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo: mocks.getUserPermissionsForServerRepo, +})); + +import { createRepoPermissionSyncWorkload } from "./repoPermissionSyncWorkload.js"; + +const repo = { + id: 42, + name: "github.com/sourcebot-dev/sourcebot", + displayName: "sourcebot-dev/sourcebot", + external_codeHostType: "github", + external_id: "123", + metadata: {}, + connections: [], +}; +const repoFindUniqueOrThrow = vi.fn().mockResolvedValue(repo); +const repoUpdate = vi.fn().mockResolvedValue(repo); +const accountFindMany = vi.fn().mockResolvedValue([]); +const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); +const permissionSyncJobUpsert = vi.fn(); +const permissionSyncJobUpdate = vi.fn().mockResolvedValue({ repo }); +const transaction = vi.fn((queries: Array>) => + Promise.all(queries), +); + +const db = { + repo: { + findUniqueOrThrow: repoFindUniqueOrThrow, + update: repoUpdate, + }, + account: { + findMany: accountFindMany, + }, + accountToRepoPermission: { + createMany: permissionCreateMany, + }, + repoPermissionSyncJob: { + upsert: permissionSyncJobUpsert, + update: permissionSyncJobUpdate, + }, + $transaction: transaction, +} as unknown as PrismaClient; + +const jobLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +const createWorkload = () => + createRepoPermissionSyncWorkload({ + db, + settings: { + maxRepoPermissionSyncJobConcurrency: 3, + } as never, + }); + +const lifecycleContext = { + data: { + repoId: 42, + }, + jobId: "job_1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: jobLogger, +}; + +const processContext = { + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.hasEntitlement.mockResolvedValue(true); + mocks.getAuthCredentialsForRepo.mockReset().mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + mocks.createOctokitFromToken.mockReset().mockResolvedValue({ octokit: {} }); + mocks.getRepoCollaborators.mockReset().mockResolvedValue([]); + repoFindUniqueOrThrow.mockResolvedValue(repo); + permissionSyncJobUpdate.mockResolvedValue({ repo }); + accountFindMany.mockResolvedValue([]); + repoUpdate.mockResolvedValue(repo); + permissionCreateMany.mockResolvedValue({ count: 0 }); +}); + +describe("repoPermissionSyncWorkload", () => { + test("uses the configured concurrency and database-backed lifecycle hooks", () => { + const workload = createWorkload(); + + expect(workload.queueSpec.name).toBe("repo-permission-sync"); + expect(workload.concurrency).toBe(3); + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("syncs the requested repo with its connections", async () => { + await createWorkload().process(processContext); + + expect(repoFindUniqueOrThrow).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + expect(mocks.getAuthCredentialsForRepo).toHaveBeenCalledWith( + repo, + jobLogger, + ); + expect(mocks.createOctokitFromToken).toHaveBeenCalledWith({ + token: "token", + url: undefined, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("does not run without the permission syncing entitlement", async () => { + mocks.hasEntitlement.mockResolvedValue(false); + + await expect(createWorkload().process(processContext)).rejects.toThrow( + "Permission syncing entitlement is not currently available.", + ); + + expect(repoFindUniqueOrThrow).not.toHaveBeenCalled(); + }); + + test("replaces all permissions for a complete GitHub sync", async () => { + const githubRepo = { + ...repo, + external_codeHostType: "github", + external_id: "123", + metadata: {}, + }; + repoFindUniqueOrThrow.mockResolvedValue(githubRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://github.com", + token: "token", + }); + const octokit = {}; + mocks.createOctokitFromToken.mockResolvedValue({ octokit }); + mocks.getRepoCollaborators.mockResolvedValue([{ id: 101 }]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(accountFindMany).toHaveBeenCalledWith({ + where: { + providerType: "github", + providerAccountId: { + in: ["101"], + }, + issuerUrl: "https://github.com", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: {}, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + expect(transaction).toHaveBeenCalledOnce(); + }); + + test("preserves account-driven permissions for a partial Bitbucket Cloud sync", async () => { + const bitbucketRepo = { + ...repo, + external_codeHostType: "bitbucketCloud", + external_id: "repo-uuid", + metadata: { + codeHostMetadata: { + bitbucketCloud: { + workspace: "sourcebot", + repoSlug: "sourcebot", + }, + }, + }, + }; + repoFindUniqueOrThrow.mockResolvedValue(bitbucketRepo); + mocks.getAuthCredentialsForRepo.mockResolvedValue({ + hostUrl: "https://bitbucket.org", + token: "token", + connectionConfig: { + user: "service-account", + }, + }); + mocks.createBitbucketCloudClient.mockReturnValue({}); + mocks.getExplicitUserPermissionsForCloudRepo.mockResolvedValue([ + { accountId: "upstream-account" }, + ]); + accountFindMany.mockResolvedValue([{ id: "account_1" }]); + + await createWorkload().process(processContext); + + expect(repoUpdate).toHaveBeenCalledWith({ + where: { id: 42 }, + data: { + permittedAccounts: { + deleteMany: { + source: "REPO_DRIVEN", + }, + }, + }, + }); + expect(permissionCreateMany).toHaveBeenCalledWith({ + data: [ + { + accountId: "account_1", + repoId: 42, + source: "REPO_DRIVEN", + }, + ], + skipDuplicates: true, + }); + }); + + test("marks a job as in progress when started", async () => { + await createWorkload().onStarted?.(lifecycleContext); + + expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ + where: { id: "job_1" }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job_1", + repoId: 42, + status: "IN_PROGRESS", + }, + }); + }); + + test("marks a job completed and updates the repo sync timestamp", async () => { + await createWorkload().onCompleted?.(lifecycleContext, undefined); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + repo: { + update: { + permissionSyncedAt: expect.any(Date), + }, + }, + }, + select: { + repo: true, + }, + }); + }); + + test("marks a job failed after terminal failure", async () => { + const error = new Error("Upstream unavailable"); + + await createWorkload().onTerminalFailure?.(lifecycleContext, error); + + expect(permissionSyncJobUpdate).toHaveBeenCalledWith({ + where: { id: "job_1" }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Upstream unavailable", + }, + select: { + repo: true, + }, + }); + expect(mocks.captureException).toHaveBeenCalledWith(error, { + tags: { + jobId: "job_1", + queue: "repo-permission-sync", + }, + }); + }); +}); diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts new file mode 100644 index 000000000..7fd1e59df --- /dev/null +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -0,0 +1,407 @@ +import * as Sentry from "@sentry/node"; +import { + PermissionSyncSource, + PrismaClient, + RepoPermissionSyncJobStatus, +} from "@sourcebot/db"; +import { + JobLogSink, + REPO_PERMISSION_SYNC_QUEUE, + repoMetadataSchema, +} from "@sourcebot/shared"; +import { hasEntitlement } from "../entitlements.js"; +import { + createOctokitFromToken, + getRepoCollaborators, + GITHUB_CLOUD_HOSTNAME, +} from "../github.js"; +import { + createGitLabFromPersonalAccessToken, + getProjectMembers, +} from "../gitlab.js"; +import { + createBitbucketCloudClient, + createBitbucketServerClient, + getExplicitUserPermissionsForCloudRepo, + getUserPermissionsForServerRepo, +} from "../bitbucket.js"; +import { + RepoAuthCredentials, + RepoWithConnections, + Settings, + Workload, +} from "../types.js"; +import { getAuthCredentialsForRepo } from "../utils.js"; +import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; + +interface RepoPermissionSyncWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createRepoPermissionSyncWorkload = ({ + db, + settings, +}: RepoPermissionSyncWorkloadDependencies): Workload<"repo-permission-sync"> => ({ + queueSpec: REPO_PERMISSION_SYNC_QUEUE, + concurrency: settings.maxRepoPermissionSyncJobConcurrency, + process: async ({ data: { repoId }, logger }) => { + if (!(await hasEntitlement("permission-syncing"))) { + throw new Error( + "Permission syncing entitlement is not currently available.", + ); + } + + const repo = await db.repo.findUniqueOrThrow({ + where: { + id: repoId, + }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + const id = repo.id; + logger.debug(`Syncing permissions for repo ${repo.displayName}...`); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + if (!credentials) { + throw new Error(`No credentials found for repo ${id}`); + } + + const { accountIds, isPartialSync = false } = + await getPermissionSyncResult({ + db, + repo, + credentials, + logger: logger, + }); + + await db.$transaction([ + db.repo.update({ + where: { + id: repo.id, + }, + data: { + permittedAccounts: { + // @note: if this is a partial sync, we only want to delete the repo-driven permissions + // since we don't want to overwrite the account-driven permissions. + deleteMany: isPartialSync + ? { + source: PermissionSyncSource.REPO_DRIVEN, + } + : {}, + }, + }, + }), + db.accountToRepoPermission.createMany({ + data: accountIds.map((accountId) => ({ + accountId, + repoId: repo.id, + source: PermissionSyncSource.REPO_DRIVEN, + })), + skipDuplicates: true, + }), + ]); + }, + onStarted: async ({ data: { repoId }, jobId }) => { + await db.repoPermissionSyncJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + status: RepoPermissionSyncJobStatus.IN_PROGRESS, + }, + }); + }, + onCompleted: async ({ jobId, logger }) => { + const { repo } = await db.repoPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + repo: { + update: { + permissionSyncedAt: new Date(), + }, + }, + }, + select: { + repo: true, + }, + }); + + logger.debug( + `Permissions synced for repo ${repo.displayName ?? repo.name}`, + ); + }, + onTerminalFailure: async ({ jobId, logger }, error) => { + Sentry.captureException(error, { + tags: { + jobId, + queue: REPO_PERMISSION_SYNC_QUEUE.name, + }, + }); + + const { repo } = await db.repoPermissionSyncJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoPermissionSyncJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + select: { + repo: true, + }, + }); + + logger.error( + `Repo permission sync job failed for repo ${repo.displayName ?? repo.name}: ${error.message}`, + ); + }, +}); + +interface ProviderPermissionSyncProps { + db: PrismaClient; + repo: RepoWithConnections; + credentials: RepoAuthCredentials; + logger: JobLogSink; +} + +interface PermissionSyncResult { + accountIds: string[]; + isPartialSync?: boolean; +} + +const getPermissionSyncResult = async ( + props: ProviderPermissionSyncProps, +): Promise => { + switch (props.repo.external_codeHostType) { + case "github": + return getGitHubPermissionSyncResult(props); + case "gitlab": + return getGitLabPermissionSyncResult(props); + case "bitbucketCloud": + return getBitbucketCloudPermissionSyncResult(props); + case "bitbucketServer": + return getBitbucketServerPermissionSyncResult(props); + default: + throw new Error( + `Unsupported code host type: ${props.repo.external_codeHostType}`, + ); + } +}; + +const getGitHubPermissionSyncResult = async ({ + db, + repo, + credentials, + logger, +}: ProviderPermissionSyncProps): Promise => { + const isGitHubCloud = credentials.hostUrl + ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME + : true; + const { octokit } = await createOctokitFromToken({ + token: credentials.token, + url: isGitHubCloud ? undefined : credentials.hostUrl, + }); + + // @note: this is a bit of a hack since the displayName _might_ not be set.. + // however, this property was introduced many versions ago and _should_ be set + // on each connection sync. Let's throw an error just in case. + if (!repo.displayName) { + throw new Error(`Repo ${repo.id} does not have a displayName`); + } + + const [owner, repoName] = repo.displayName.split("/"); + const collaborators = await getRepoCollaborators(owner, repoName, octokit); + const githubUserIds = collaborators.map((collaborator) => + collaborator.id.toString(), + ); + + logger.debug(`Found ${collaborators.length} collaborator(s)`, { + collaborators: collaborators.flatMap(({ email, login }) => ({email, login})), + }); + + const accounts = await db.account.findMany({ + where: { + providerType: "github", + providerAccountId: { + in: githubUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getGitLabPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const api = await createGitLabFromPersonalAccessToken({ + token: credentials.token, + url: credentials.hostUrl, + }); + + const projectId = repo.external_id; + if (!projectId) { + throw new Error(`Repo ${repo.id} does not have an external_id`); + } + + const members = await getProjectMembers(projectId, api); + const gitlabUserIds = members.map((member) => member.id.toString()); + + const accounts = await db.account.findMany({ + where: { + providerType: "gitlab", + providerAccountId: { + in: gitlabUserIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + }; +}; + +const getBitbucketCloudPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const config = credentials.connectionConfig as + | BitbucketConnectionConfig + | undefined; + if (!config) { + throw new Error(`No connection config found for repo ${repo.id}`); + } + + const client = createBitbucketCloudClient(config.user, credentials.token); + + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketCloudMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketCloud; + if (!bitbucketCloudMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`, + ); + } + + const { workspace, repoSlug } = bitbucketCloudMetadata; + + // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* + // granted access to this repository. Users who have access via a group added to the repo, + // via project-level membership, or via a group in a project are NOT captured here. + // These users will still gain access through account-driven permission syncing, + // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before + // they see the repository in Sourcebot. + // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get + const users = await getExplicitUserPermissionsForCloudRepo( + client, + workspace, + repoSlug, + ); + const userAccountIds = users.map((user) => user.accountId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-cloud", + providerAccountId: { + in: userAccountIds, + }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + // Since we only fetch users who have been explicitly granted access to the repo, + // this is a partial sync. + isPartialSync: true, + }; +}; + +const getBitbucketServerPermissionSyncResult = async ({ + db, + repo, + credentials, +}: ProviderPermissionSyncProps): Promise => { + const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); + if (!parsedMetadata.success) { + throw new Error( + `Repo ${repo.id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`, + ); + } + const bitbucketServerMetadata = + parsedMetadata.data.codeHostMetadata?.bitbucketServer; + if (!bitbucketServerMetadata) { + throw new Error( + `Repo ${repo.id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`, + ); + } + + const { projectKey, repoSlug } = bitbucketServerMetadata; + const hostUrl = credentials.hostUrl; + + if (!hostUrl) { + throw new Error( + `No host URL found for Bitbucket Server repo ${repo.id}`, + ); + } + + // @note: This covers users with direct repo-level and project-level permissions. + // Users with access only via groups are NOT captured here. Those users will + // still gain access through account-driven permission syncing. + const client = createBitbucketServerClient( + hostUrl, + /* user = */ undefined, + credentials.token, + ); + const users = await getUserPermissionsForServerRepo( + client, + projectKey, + repoSlug, + ); + const userIds = users.map((user) => user.userId); + + const accounts = await db.account.findMany({ + where: { + providerType: "bitbucket-server", + providerAccountId: { in: userIds }, + issuerUrl: credentials.hostUrl, + }, + }); + + return { + accountIds: accounts.map((account) => account.id), + isPartialSync: true, + }; +}; diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts deleted file mode 100644 index 536f48a09..000000000 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ /dev/null @@ -1,436 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { PermissionSyncSource, PrismaClient, Repo, RepoPermissionSyncJobStatus } from "@sourcebot/db"; -import { createLogger, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES } from "@sourcebot/shared"; -import { env } from "@sourcebot/shared"; -import { hasEntitlement } from "../entitlements.js"; -import { DelayedError, Job, Queue, Worker } from 'bullmq'; -import { Redis } from 'ioredis'; -import { createOctokitFromToken, getRepoCollaborators, GITHUB_CLOUD_HOSTNAME } from "../github.js"; -import { createGitLabFromPersonalAccessToken, getProjectMembers } from "../gitlab.js"; -import { createBitbucketCloudClient, createBitbucketServerClient, getExplicitUserPermissionsForCloudRepo, getUserPermissionsForServerRepo } from "../bitbucket.js"; -import { repoMetadataSchema } from "@sourcebot/shared"; -import { Settings } from "../types.js"; -import { getAuthCredentialsForRepo, setIntervalAsync } from "../utils.js"; -import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/index.type"; - -type RepoPermissionSyncJob = { - jobId: string; -} - -const QUEUE_NAME = 'repoPermissionSyncQueue'; -const POLLING_INTERVAL_MS = 1000; -const ENTITLEMENT_RETRY_DELAY_MS = 30 * 1000; -const LOG_TAG = 'repo-permission-syncer'; - -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); - -export class RepoPermissionSyncer { - private queue: Queue; - private worker: Worker; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - ) { - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - }); - this.worker = new Worker(QUEUE_NAME, this.runJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoPermissionSyncJobConcurrency, - }); - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobFailed.bind(this)); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - - this.interval = setIntervalAsync(async () => { - if (!await hasEntitlement('permission-syncing')) { - return; - } - - // @todo: make this configurable - const thresholdDate = new Date(Date.now() - this.settings.repoDrivenPermissionSyncIntervalMs); - - const repos = await this.db.repo.findMany({ - // Repos need their permissions to be synced against the code host when... - where: { - AND: [ - // They are not public. Public repositories are always visible to all users, therefore we don't - // need to explicitly perform permission syncing for them. - // @see: packages/web/src/prisma.ts - { - isPublic: false - }, - // They belong to a code host that supports permissions syncing - { - external_codeHostType: { - in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, - } - }, - // They have at least one connection with permission enforcement enabled - { - connections: { - some: { - connection: { - enforcePermissions: true, - } - } - } - }, - // They have not been synced within the threshold date. - { - OR: [ - { permissionSyncedAt: null }, - { permissionSyncedAt: { lt: thresholdDate } }, - ], - }, - // There aren't any active or recently failed jobs. - { - NOT: { - permissionSyncJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs - { - status: { - in: [ - RepoPermissionSyncJobStatus.PENDING, - RepoPermissionSyncJobStatus.IN_PROGRESS, - ], - } - }, - // Don't schedule if there are recent failed jobs (within the threshold date). Note `gt` is used here since this is a inverse condition. - { - AND: [ - { status: RepoPermissionSyncJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - } - } - }, - ] - } - }); - - await this.schedulePermissionSync(repos); - }, POLLING_INTERVAL_MS); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.worker.close(/* force = */ true); - await this.queue.close(); - } - - private async schedulePermissionSync(repos: Repo[]) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoPermissionSyncJob.createManyAndReturn({ - data: repos.map(repo => ({ - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - await this.queue.addBulk(jobs.map((job) => ({ - name: 'repoPermissionSyncJob', - data: { - jobId: job.id, - }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - // Priority 1 (high) for never-synced, Priority 2 (normal) for re-sync - priority: job.repo.permissionSyncedAt === null ? 1 : 2, - } - }))) - } - - private async runJob(job: Job) { - if (!await hasEntitlement('permission-syncing')) { - await job.moveToDelayed(Date.now() + ENTITLEMENT_RETRY_DELAY_MS, job.token); - throw new DelayedError('Permission syncing entitlement is not currently available.'); - } - - const id = job.data.jobId; - const logger = createJobLogger(id); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id, - }, - data: { - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - }, - select: { - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - if (!repo) { - throw new Error(`Repo ${id} not found`); - } - - logger.debug(`Syncing permissions for repo ${repo.displayName}...`); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - if (!credentials) { - throw new Error(`No credentials found for repo ${id}`); - } - - const { - accountIds, - isPartialSync = false, - } = await (async (): Promise<{ - accountIds: string[], - isPartialSync?: boolean - }> => { - if (repo.external_codeHostType === 'github') { - const isGitHubCloud = credentials.hostUrl ? new URL(credentials.hostUrl).hostname === GITHUB_CLOUD_HOSTNAME : true; - const { octokit } = await createOctokitFromToken({ - token: credentials.token, - url: isGitHubCloud ? undefined : credentials.hostUrl, - }); - - // @note: this is a bit of a hack since the displayName _might_ not be set.. - // however, this property was introduced many versions ago and _should_ be set - // on each connection sync. Let's throw an error just in case. - if (!repo.displayName) { - throw new Error(`Repo ${id} does not have a displayName`); - } - - const [owner, repoName] = repo.displayName.split('/'); - - const collaborators = await getRepoCollaborators(owner, repoName, octokit); - const githubUserIds = collaborators.map(collaborator => collaborator.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'github', - providerAccountId: { - in: githubUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'gitlab') { - const api = await createGitLabFromPersonalAccessToken({ - token: credentials.token, - url: credentials.hostUrl, - }); - - const projectId = repo.external_id; - if (!projectId) { - throw new Error(`Repo ${id} does not have an external_id`); - } - - const members = await getProjectMembers(projectId, api); - const gitlabUserIds = members.map(member => member.id.toString()); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'gitlab', - providerAccountId: { - in: gitlabUserIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - } - } else if (repo.external_codeHostType === 'bitbucketCloud') { - const config = credentials.connectionConfig as BitbucketConnectionConfig | undefined; - if (!config) { - throw new Error(`No connection config found for repo ${id}`); - } - - const client = createBitbucketCloudClient(config.user, credentials.token); - - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketCloudMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketCloud; - if (!bitbucketCloudMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Cloud metadata (workspace/repoSlug)`); - } - - const { workspace, repoSlug } = bitbucketCloudMetadata; - - // @note: The Bitbucket Cloud permissions API only returns users who have been *directly* - // granted access to this repository. Users who have access via a group added to the repo, - // via project-level membership, or via a group in a project are NOT captured here. - // These users will still gain access through user-driven syncing (accountPermissionSyncer), - // but there may be a delay of up to `userDrivenPermissionSyncIntervalMs` before - // they see the repository in Sourcebot. - // @see: https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/#api-repositories-workspace-repo-slug-permissions-config-users-get - const users = await getExplicitUserPermissionsForCloudRepo(client, workspace, repoSlug); - const userAccountIds = users.map(u => u.accountId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-cloud', - providerAccountId: { - in: userAccountIds, - }, - issuerUrl: credentials.hostUrl, - }, - }); - - return { - accountIds: accounts.map(account => account.id), - // Since we only fetch users who have been explicitly granted access to the repo, - // this is a partial sync. - isPartialSync: true, - } - } else if (repo.external_codeHostType === 'bitbucketServer') { - const parsedMetadata = repoMetadataSchema.safeParse(repo.metadata); - if (!parsedMetadata.success) { - throw new Error(`Repo ${id} has invalid metadata: ${JSON.stringify(parsedMetadata.error.errors)}`); - } - const bitbucketServerMetadata = parsedMetadata.data.codeHostMetadata?.bitbucketServer; - if (!bitbucketServerMetadata) { - throw new Error(`Repo ${id} is missing required Bitbucket Server metadata (projectKey/repoSlug)`); - } - - const { projectKey, repoSlug } = bitbucketServerMetadata; - const hostUrl = credentials.hostUrl; - - if (!hostUrl) { - throw new Error(`No host URL found for Bitbucket Server repo ${id}`); - } - - // @note: This covers users with direct repo-level and project-level permissions. - // Users with access only via groups are NOT captured here. Those users will - // still gain access through account-driven syncing (accountPermissionSyncer). - const client = createBitbucketServerClient(hostUrl, /* user = */ undefined, credentials.token); - const users = await getUserPermissionsForServerRepo(client, projectKey, repoSlug); - const userIds = users.map(u => u.userId); - - const accounts = await this.db.account.findMany({ - where: { - providerType: 'bitbucket-server', - providerAccountId: { in: userIds }, - issuerUrl: credentials.hostUrl, - } - }); - - return { - accountIds: accounts.map(account => account.id), - isPartialSync: true, - } - } - - throw new Error(`Unsupported code host type: ${repo.external_codeHostType}`); - })(); - - await this.db.$transaction([ - this.db.repo.update({ - where: { - id: repo.id, - }, - data: { - permittedAccounts: { - // @note: if this is a partial sync, we only want to delete the repo-driven permissions - // since we don't want to overwrite the account-driven permissions. - deleteMany: isPartialSync ? { - source: PermissionSyncSource.REPO_DRIVEN, - } : {}, - } - } - }), - this.db.accountToRepoPermission.createMany({ - data: accountIds.map(accountId => ({ - accountId, - repoId: repo.id, - source: PermissionSyncSource.REPO_DRIVEN, - })), - skipDuplicates: true, - }) - ]); - } - - private async onJobCompleted(job: Job) { - const logger = createJobLogger(job.data.jobId); - - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.COMPLETED, - repo: { - update: { - permissionSyncedAt: new Date(), - } - }, - completedAt: new Date(), - }, - select: { - repo: true - } - }); - - logger.debug(`Permissions synced for repo ${repo.displayName ?? repo.name}`); - } - - private async onJobFailed(job: Job | undefined, err: Error) { - const logger = createJobLogger(job?.data.jobId ?? 'unknown'); - - Sentry.captureException(err, { - tags: { - jobId: job?.data.jobId, - queue: QUEUE_NAME, - } - }); - - const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err.message}`; - - if (job) { - const { repo } = await this.db.repoPermissionSyncJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: err.message, - }, - select: { - repo: true - }, - }); - logger.error(errorMessage(repo.displayName ?? repo.name)); - } else { - logger.error(errorMessage('unknown repo (id not found)')); - } - } -} diff --git a/packages/backend/src/ee/syncSearchContexts.test.ts b/packages/backend/src/ee/syncSearchContexts.test.ts index 9aa1decfd..89ffec5d6 100644 --- a/packages/backend/src/ee/syncSearchContexts.test.ts +++ b/packages/backend/src/ee/syncSearchContexts.test.ts @@ -21,6 +21,17 @@ vi.mock('../entitlements.js', () => ({ getPlan: vi.fn(() => Promise.resolve('enterprise')), })); +// `syncSearchContexts` imports the prisma singleton, which builds a real client (and needs +// DATABASE_URL) at import time. Stand in for it with an object that `buildDb` re-populates +// with fresh mocks for each test, so tests stay isolated from one another. +const { prismaMock } = vi.hoisted(() => ({ + prismaMock: {} as Record, +})); + +vi.mock('../prisma.js', () => ({ + prisma: prismaMock, +})); + import { syncSearchContexts } from './syncSearchContexts.js'; // Helper to build a repo record with GitLab topics stored in metadata. @@ -64,20 +75,25 @@ const buildDb = (overrides: Partial<{ connectionFindMany: unknown[]; searchContextFindUnique: unknown; searchContextFindMany: unknown[]; -}> = {}): PrismaClient => ({ - repo: { - findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), - }, - connection: { - findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), - }, - searchContext: { - findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), - findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), - upsert: vi.fn().mockResolvedValue({}), - delete: vi.fn().mockResolvedValue({}), - }, -} as unknown as PrismaClient); +}> = {}): PrismaClient => { + // Overwrite every delegate so no mock survives from a previous test. + Object.assign(prismaMock, { + repo: { + findMany: vi.fn().mockResolvedValue(overrides.repoFindMany ?? []), + }, + connection: { + findMany: vi.fn().mockResolvedValue(overrides.connectionFindMany ?? []), + }, + searchContext: { + findUnique: vi.fn().mockResolvedValue(overrides.searchContextFindUnique ?? null), + findMany: vi.fn().mockResolvedValue(overrides.searchContextFindMany ?? []), + upsert: vi.fn().mockResolvedValue({}), + delete: vi.fn().mockResolvedValue({}), + }, + }); + + return prismaMock as unknown as PrismaClient; +}; describe('syncSearchContexts - includeTopics', () => { test('includes repos whose topics match an includeTopics entry', async () => { @@ -90,7 +106,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -109,7 +124,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -127,7 +141,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend', 'core'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -146,7 +159,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -165,7 +177,6 @@ describe('syncSearchContexts - includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -188,7 +199,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -209,7 +219,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -230,7 +239,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -251,7 +259,6 @@ describe('syncSearchContexts - excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -274,7 +281,6 @@ describe('syncSearchContexts - includeTopics + excludeTopics combined', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -298,7 +304,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -319,7 +324,6 @@ describe('syncSearchContexts - includeTopics combined with include globs', () => }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -340,7 +344,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -359,7 +362,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['core-*'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -377,7 +379,6 @@ describe('syncSearchContexts - GitHub includeTopics', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -400,7 +401,6 @@ describe('syncSearchContexts - GitHub excludeTopics', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -422,7 +422,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { myContext: { includeTopics: ['backend'] }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; @@ -446,7 +445,6 @@ describe('syncSearchContexts - mixed GitHub and GitLab repos', () => { }, }, orgId: 1, - db, }); const upsertCall = vi.mocked(db.searchContext.upsert).mock.calls[0][0]; diff --git a/packages/backend/src/ee/syncSearchContexts.ts b/packages/backend/src/ee/syncSearchContexts.ts index 6a02c9062..a3592a30f 100644 --- a/packages/backend/src/ee/syncSearchContexts.ts +++ b/packages/backend/src/ee/syncSearchContexts.ts @@ -1,20 +1,19 @@ import micromatch from "micromatch"; import { createLogger } from "@sourcebot/shared"; -import { PrismaClient } from "@sourcebot/db"; import { repoMetadataSchema, SOURCEBOT_SUPPORT_EMAIL } from "@sourcebot/shared"; import { hasEntitlement } from "../entitlements.js"; import { SearchContext } from "@sourcebot/schemas/v3/index.type"; +import { prisma } from "../prisma.js"; const logger = createLogger('sync-search-contexts'); interface SyncSearchContextsParams { contexts?: { [key: string]: SearchContext } | undefined; orgId: number; - db: PrismaClient; } export const syncSearchContexts = async (params: SyncSearchContextsParams) => { - const { contexts, orgId, db } = params; + const { contexts, orgId } = params; if (!await hasEntitlement("search-contexts")) { if (contexts) { @@ -25,7 +24,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { if (contexts) { for (const [key, newContextConfig] of Object.entries(contexts)) { - const allRepos = await db.repo.findMany({ + const allRepos = await prisma.repo.findMany({ where: { orgId, }, @@ -44,7 +43,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if(newContextConfig.includeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -101,7 +100,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } if (newContextConfig.excludeConnections) { - const connections = await db.connection.findMany({ + const connections = await prisma.connection.findMany({ where: { orgId, name: { @@ -145,7 +144,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { }); } - const currentReposInContext = (await db.searchContext.findUnique({ + const currentReposInContext = (await prisma.searchContext.findUnique({ where: { name_orgId: { name: key, @@ -157,7 +156,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } }))?.repos ?? []; - await db.searchContext.upsert({ + await prisma.searchContext.upsert({ where: { name_orgId: { name: key, @@ -195,7 +194,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { } } - const deletedContexts = await db.searchContext.findMany({ + const deletedContexts = await prisma.searchContext.findMany({ where: { name: { notIn: Object.keys(contexts ?? {}), @@ -206,7 +205,7 @@ export const syncSearchContexts = async (params: SyncSearchContextsParams) => { for (const context of deletedContexts) { logger.debug(`Deleting search context with name '${context.name}'. ID: ${context.id}`); - await db.searchContext.delete({ + await prisma.searchContext.delete({ where: { id: context.id, } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index df2a18993..f53dee93a 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -2,22 +2,22 @@ import "./instrument.js"; import * as Sentry from "@sentry/node"; import { createLogger, env, getConfigSettings } from "@sourcebot/shared"; -import { prisma } from "./prisma.js"; import 'express-async-errors'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; -import { Api } from "./api.js"; -import { AttachmentPruner } from "./attachmentPruner.js"; import { ConfigManager } from "./configManager.js"; -import { ConnectionManager } from './connectionManager.js'; import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, SHUTDOWN_SIGNALS } from './constants.js'; -import { AccountPermissionSyncer } from "./ee/accountPermissionSyncer.js"; -import { AuditLogPruner } from "./ee/auditLogPruner.js"; -import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; +import { BullMQJobManager } from "./jobManager.js"; import { shutdownPosthog } from "./posthog.js"; +import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; -import { RepoIndexManager } from "./repoIndexManager.js"; +import { createReconciliationWorkload } from "./reconciliationWorkload.js"; import { redis } from "./redis.js"; +import { createConnectionWorkload } from "./connectionWorkload.js"; +import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; +import { Api } from "./api.js"; +import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; +import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; const logger = createLogger('backend-entrypoint'); @@ -40,40 +40,50 @@ try { process.exit(1); } -const promClient = new PromClient(); const settings = await getConfigSettings(env.CONFIG_PATH); -const connectionManager = new ConnectionManager(prisma, settings, redis, promClient); -const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); -const accountPermissionSyncer = new AccountPermissionSyncer(prisma, settings, redis); -const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClient); -const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH); -const auditLogPruner = new AuditLogPruner(prisma); -const attachmentPruner = new AttachmentPruner(prisma); - -connectionManager.startScheduler(); -await repoIndexManager.startScheduler(); -auditLogPruner.startScheduler(); -attachmentPruner.startScheduler(); - -if (env.PERMISSION_SYNC_ENABLED === 'true') { - if (env.PERMISSION_SYNC_REPO_DRIVEN_ENABLED === 'true') { - await repoPermissionSyncer.startScheduler(); - } - await accountPermissionSyncer.startScheduler(); -} - -const api = new Api( - promClient, - prisma, - connectionManager, - repoIndexManager, - accountPermissionSyncer, -); +const promClient = new PromClient(); logger.info('Worker started.'); +const jobManager = new BullMQJobManager(redis); + +const reconciliationWorkload = createReconciliationWorkload({ + db: prisma, + settings, +}); +const connectionWorkload = createConnectionWorkload({ + db: prisma, + settings, +}); +const repoIndexWorkload = createRepoIndexWorkload({ + db: prisma, + settings, +}); +const accountPermissionSyncWorkload = createAccountPermissionSyncWorkload({ + db: prisma, + settings, +}); +const repoPermissionSyncWorkload = createRepoPermissionSyncWorkload({ + db: prisma, + settings, +}); + +jobManager.register(reconciliationWorkload); +jobManager.register(connectionWorkload); +jobManager.register(repoIndexWorkload); +jobManager.register(accountPermissionSyncWorkload); +jobManager.register(repoPermissionSyncWorkload); + +const api = new Api(promClient, jobManager.getQueues()); + +await cleanupOrphanedRepoResources(prisma); +await jobManager.start(); + +const configManager = new ConfigManager(jobManager, env.CONFIG_PATH); + + const listenToShutdownSignals = () => { const signals = SHUTDOWN_SIGNALS; @@ -88,13 +98,8 @@ const listenToShutdownSignals = () => { logger.info(`Received ${signal}, cleaning up...`); - await repoIndexManager.dispose() - await connectionManager.dispose() - await repoPermissionSyncer.dispose() - await accountPermissionSyncer.dispose() - await auditLogPruner.dispose() - await attachmentPruner.dispose() await configManager.dispose() + await jobManager.stop(); await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts new file mode 100644 index 000000000..5421c8a4b --- /dev/null +++ b/packages/backend/src/jobManager.test.ts @@ -0,0 +1,279 @@ +import { Redis } from "ioredis"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { ProcessContext, Workload } from "./types.js"; + +const mocks = vi.hoisted(() => { + const jobLogger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + flush: vi.fn(), + }; + return { + enqueue: vi.fn(), + producerClose: vi.fn(), + workerClose: vi.fn(), + jobLogger, + createBullMQJobLogger: vi.fn(() => jobLogger), + workers: [] as Array<{ + processor: (job: unknown) => Promise; + handlers: Map void>; + }>, + }; +}); + +// The module under test creates a logger at import time; stub it so importing pure helpers +// has no side effects (mirrors repoIndexManager.test.ts). +vi.mock("@sourcebot/shared", () => ({ + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + createBullMQJobLogger: mocks.createBullMQJobLogger, + BullMQClient: class { + enqueue = mocks.enqueue; + close = mocks.producerClose; + getQueue = vi.fn(() => ({ + getJobCounts: vi.fn(), + upsertJobScheduler: vi.fn(), + })); + }, +})); + +// Mock the constants module directly so its env-derived cache-dir paths don't load. +vi.mock("./constants.js", () => ({ + WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, +})); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("bullmq", () => ({ + Worker: class { + private readonly record: (typeof mocks.workers)[number]; + + constructor( + _name: string, + processor: (job: unknown) => Promise, + ) { + this.record = { processor, handlers: new Map() }; + mocks.workers.push(this.record); + } + + on(event: string, handler: (...args: unknown[]) => void) { + this.record.handlers.set(event, handler); + } + + close = mocks.workerClose; + }, +})); + +import { + BullMQJobManager, + normalizeJobState, + parseDuration, +} from "./jobManager.js"; + +describe("parseDuration", () => { + test.each([ + ["500ms", 500], + ["30s", 30_000], + ["5m", 300_000], + ["6h", 21_600_000], + ["1d", 86_400_000], + ])("parses %s", (input, expected) => { + expect(parseDuration(input)).toBe(expected); + }); + + test("trims surrounding whitespace", () => { + expect(parseDuration(" 10m ")).toBe(600_000); + }); + + test.each(["", "5", "m", "5x", "1.5h", "-5m", "5 m"])( + 'throws on malformed "%s"', + (input) => { + expect(() => parseDuration(input)).toThrow(); + }, + ); +}); + +describe("normalizeJobState", () => { + test.each([ + "waiting", + "active", + "delayed", + "completed", + "failed", + "paused", + ])('passes through "%s"', (state) => { + expect(normalizeJobState(state)).toBe(state); + }); + + test("collapses prioritized and waiting-children to waiting", () => { + expect(normalizeJobState("prioritized")).toBe("waiting"); + expect(normalizeJobState("waiting-children")).toBe("waiting"); + }); + + test("maps anything unrecognized to unknown", () => { + expect(normalizeJobState("something-else")).toBe("unknown"); + expect(normalizeJobState("unknown")).toBe("unknown"); + }); +}); + +const createWorkload = ( + overrides: Partial> = {}, +): Workload<"connection-sync", { repoCount: number }> => ({ + queueSpec: { + name: "connection-sync", + dedupKey: ({ connectionId }) => `connection:${connectionId}`, + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, + concurrency: 2, + process: vi.fn(async () => ({ repoCount: 3 })), + ...overrides, +}); + +const data = { connectionId: 42, orgId: 1 }; +const job = { + id: "job-1", + queueName: "connection-sync", + data, + attemptsMade: 2, + opts: { attempts: 2 }, + log: vi.fn(), + updateProgress: vi.fn(), +}; + +describe("BullMQJobManager lifecycle", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.workers.length = 0; + mocks.enqueue.mockResolvedValue("job-1"); + }); + + test("delegates enqueueing to BullMQClient and returns its job id", async () => { + const manager = new BullMQJobManager({} as Redis); + const workload = createWorkload(); + manager.register(workload); + + const result = await manager.trigger("connection-sync", data); + + expect(result).toBe("job-1"); + expect(mocks.enqueue).toHaveBeenCalledWith(workload.queueSpec, data); + }); + + test("calls onStarted before processing and onCompleted after completion", async () => { + const calls: string[] = []; + const workload = createWorkload({ + onStarted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle started"); + calls.push("started"); + }), + process: vi.fn(async () => { + calls.push("processed"); + return { repoCount: 3 }; + }), + onCompleted: vi.fn(async ({ logger }) => { + logger.info("Lifecycle completed"); + calls.push("completed"); + }), + }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const result = await mocks.workers[0].processor({ + ...job, + attemptsMade: 0, + }); + expect(calls).toEqual(["started", "processed"]); + + mocks.workers[0].handlers.get("completed")?.(job, result); + await vi.waitFor(() => + expect(calls).toEqual(["started", "processed", "completed"]), + ); + expect(workload.onCompleted).toHaveBeenCalledWith( + expect.objectContaining({ + data, + jobId: "job-1", + maxAttempts: 2, + logger: mocks.jobLogger, + }), + { repoCount: 3 }, + ); + expect(mocks.jobLogger.info).toHaveBeenCalledWith("Lifecycle started"); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Lifecycle completed", + ); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledTimes(2); + }); + }); + + test("provides the structured job logger to the workload processor", async () => { + const process = vi.fn( + async (context: ProcessContext<"connection-sync">) => { + context.logger.info("Processing connection"); + return { repoCount: 3 }; + }, + ); + const manager = new BullMQJobManager({} as Redis); + manager.register(createWorkload({ process })); + await manager.start(); + + await mocks.workers[0].processor({ ...job, attemptsMade: 0 }); + + expect(process).toHaveBeenCalledWith( + expect.objectContaining({ + logger: mocks.jobLogger, + }), + ); + expect(mocks.jobLogger.info).toHaveBeenCalledWith( + "Processing connection", + ); + expect(mocks.jobLogger.flush).toHaveBeenCalled(); + }); + + test("reports lifecycle metadata after terminal failure", async () => { + const onTerminalFailure = vi.fn(async ({ logger }) => { + logger.error("Lifecycle failed"); + }); + const workload = createWorkload({ onTerminalFailure }); + const manager = new BullMQJobManager({} as Redis); + manager.register(workload); + await manager.start(); + + const error = new Error("failed"); + mocks.workers[0].handlers.get("failed")?.(job, error); + + await vi.waitFor(() => { + expect(onTerminalFailure).toHaveBeenCalledWith( + expect.objectContaining({ + data, + jobId: "job-1", + attemptsMade: 2, + maxAttempts: 2, + logger: mocks.jobLogger, + }), + error, + ); + }); + expect(mocks.jobLogger.error).toHaveBeenCalledWith("Lifecycle failed"); + await vi.waitFor(() => { + expect(mocks.jobLogger.flush).toHaveBeenCalledOnce(); + }); + expect(mocks.createBullMQJobLogger).toHaveBeenCalledWith( + expect.objectContaining({ id: "job-1", attemptsMade: 2 }), + expect.objectContaining({ attempt: 2 }), + ); + }); +}); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts new file mode 100644 index 000000000..a617a5812 --- /dev/null +++ b/packages/backend/src/jobManager.ts @@ -0,0 +1,318 @@ +import * as Sentry from "@sentry/node"; +import { + BullMQClient, + createBullMQJobLogger, + createLogger, + DataOf, + JobLogSink, + QueueName, +} from "@sourcebot/shared"; +import { Job, Queue, Worker } from "bullmq"; +import { Redis } from "ioredis"; +import { WORKER_STOP_GRACEFUL_TIMEOUT_MS } from "./constants.js"; +import { + JobDetail, + JobManager, + Schedule, + JobLifecycleContext, + Workload, +} from "./types.js"; +import { prisma } from "./prisma.js"; + +const LOG_TAG = "job-manager"; +const logger = createLogger(LOG_TAG); + +const DURATION_UNITS_MS: Record = { + ms: 1, + s: 1000, + m: 1000 * 60, + h: 1000 * 60 * 60, + d: 1000 * 60 * 60 * 24, +}; + +export const parseDuration = (value: string): number => { + const match = /^(\d+)(ms|s|m|h|d)$/.exec(value.trim()); + if (!match) { + throw new Error( + `Invalid duration "${value}". Expected e.g. "500ms", "30s", "5m", "6h", "1d".`, + ); + } + return Number(match[1]) * DURATION_UNITS_MS[match[2]]; +}; + +export const normalizeJobState = (state: string): JobDetail["state"] => { + switch (state) { + case "waiting": + case "active": + case "delayed": + case "completed": + case "failed": + case "paused": + return state; + case "prioritized": + case "waiting-children": + return "waiting"; + default: + return "unknown"; + } +}; + +const scheduleToRepeat = (schedule: Schedule) => + "pattern" in schedule + ? { pattern: schedule.pattern } + : { every: parseDuration(schedule.every) }; + +export class BullMQJobManager implements JobManager { + private readonly workloads = new Map< + string, + Workload + >(); + private readonly workers = new Map(); + private readonly bullmqClient: BullMQClient; + private readonly abortController = new AbortController(); + + constructor(private readonly connection: Redis) { + this.bullmqClient = new BullMQClient(connection); + } + + register(workload: Workload): void { + const name = workload.queueSpec.name; + if (this.workloads.has(name)) { + throw new Error(`Workload "${name}" is already registered`); + } + this.workloads.set(name, workload); + } + + getQueues(): Queue[] { + return [...this.workloads.values()].map((workload) => + this.bullmqClient.getQueue(workload.queueSpec), + ); + } + + async start(): Promise { + if (this.workloads.size === 0) { + logger.debug( + "start() called with nothing registered; nothing to do", + ); + return; + } + + for (const workload of this.workloads.values()) { + await this.startWorkload(workload); + } + + logger.info( + `Started ${this.workloads.size} workload(s) [${[...this.workloads.keys()].join(", ")}]`, + ); + } + + async trigger( + workloadName: TName, + data: DataOf, + ): Promise { + const workload = this.workloads.get(workloadName) as + | Workload + | undefined; + if (!workload) { + throw new Error( + `Cannot trigger unknown workload "${workloadName}"`, + ); + } + return this.bullmqClient.enqueue(workload.queueSpec, data); + } + + async stop(): Promise { + this.abortController.abort(); + + await Promise.all( + [...this.workers.values()].map((worker) => + Promise.race([ + worker.close(), + new Promise((resolve) => + setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS), + ), + ]), + ), + ); + + await this.bullmqClient.close(); + + logger.info("Job manager stopped"); + } + + private async startWorkload( + workload: Workload, + ): Promise { + const { queueSpec: spec, concurrency, rateLimit, schedule } = workload; + + const queue = this.bullmqClient.getQueue(spec); + + const worker = new Worker( + spec.name, + async (job) => { + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${spec.name}:job:${job.id ?? "unknown"}`, + }); + const lifecycleContext = this.jobLifecycleContext( + job, + jobLogger, + ); + + try { + await workload.onStarted?.(lifecycleContext); + const result = await workload.process({ + ...lifecycleContext, + signal: this.abortController.signal, + updateProgress: (progress) => + job.updateProgress(progress), + trigger: (target, data) => this.trigger(target, data), + }); + return result; + } catch (error) { + jobLogger.error( + `Workload "${spec.name}" attempt failed`, + error, + ); + throw error; + } finally { + await jobLogger.flush(); + } + }, + { + connection: this.connection, + concurrency, + maxStalledCount: 1, + ...(rateLimit + ? { + limiter: { + max: rateLimit.max, + duration: parseDuration(rateLimit.per), + }, + } + : {}), + }, + ); + + worker.on("failed", (job, error) => { + void this.onWorkloadJobFailed(workload, job, error); + }); + worker.on("completed", (job, result) => { + void this.onWorkloadJobCompleted(workload, job, result); + }); + worker.on("error", (error) => { + logger.error(`Worker "${spec.name}" error:`, error); + }); + + this.workers.set(spec.name, worker); + + if (schedule) { + // @note: jobs produced by BullMQ's scheduler bypass the deduplication check that + // `Queue.add` goes through, so a dedup key would be silently ignored here. A + // scheduled workload gets its overlap protection from `concurrency` instead: the + // next tick's job is only created once the current one goes active, so at most one + // run is ever queued behind the one in flight. + await queue.upsertJobScheduler( + `schedule:${spec.name}`, + scheduleToRepeat(schedule), + { + name: spec.name, + opts: { + attempts: spec.jobOptions.attempts, + removeOnComplete: { + count: spec.jobOptions.keep.completed, + }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + keepLogs: spec.jobOptions.keepLogs, + }, + }, + ); + } + } + + private async onWorkloadJobFailed( + workload: Workload, + job: Job | undefined, + error: Error, + ): Promise { + if (!job) { + return; + } + const maxAttempts = job.opts.attempts ?? 1; + const isTerminal = job.attemptsMade >= maxAttempts; + if (!isTerminal) { + logger.warn( + `Workload "${workload.queueSpec.name}" job ${job.id} failed attempt ${job.attemptsMade}/${maxAttempts}; will retry: ${error.message}`, + ); + return; + } + logger.error( + `Workload "${workload.queueSpec.name}" job ${job.id} failed terminally after ${job.attemptsMade} attempt(s): ${error.message}`, + ); + + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); + try { + await workload.onTerminalFailure?.( + this.jobLifecycleContext(job, jobLogger), + error, + ); + } catch (hookError) { + Sentry.captureException(hookError); + jobLogger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onTerminalFailure for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); + } + } + + private async onWorkloadJobCompleted( + workload: Workload, + job: Job, + result: TResult, + ): Promise { + const jobLogger = createBullMQJobLogger(job, { + label: `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`, + attempt: Math.max(job.attemptsMade, 1), + }); + try { + await workload.onCompleted?.( + this.jobLifecycleContext(job, jobLogger), + result, + ); + } catch (hookError) { + Sentry.captureException(hookError); + jobLogger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw`, + hookError, + ); + logger.error( + `onCompleted for workload "${workload.queueSpec.name}" threw:`, + hookError, + ); + } finally { + await jobLogger.flush(); + } + } + + private jobLifecycleContext( + job: Job, + logger: JobLogSink, + ): JobLifecycleContext { + return { + data: job.data, + jobId: job.id ?? "", + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts ?? 1, + prisma, + logger, + }; + } +} diff --git a/packages/backend/src/reconciliationWorkload.test.ts b/packages/backend/src/reconciliationWorkload.test.ts new file mode 100644 index 000000000..2391cc6ee --- /dev/null +++ b/packages/backend/src/reconciliationWorkload.test.ts @@ -0,0 +1,375 @@ +import type { PrismaClient } from '@sourcebot/db'; +import type { JobLogger } from '@sourcebot/shared'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + env: { + PERMISSION_SYNC_ENABLED: 'true', + }, + hasEntitlement: vi.fn(), +})); + +vi.mock('@sourcebot/shared', () => ({ + env: mocks.env, + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ + 'github', + 'gitlab', + 'bitbucket-cloud', + 'bitbucket-server', + ], + PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES: [ + 'github', + 'gitlab', + 'bitbucketCloud', + 'bitbucketServer', + ], + RECONCILIATION_QUEUE: { + name: 'reconciliation', + jobOptions: { + attempts: 2, + backoff: { type: 'exponential', delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: 500, + }, + }, +})); + +vi.mock('./entitlements.js', () => ({ + hasEntitlement: mocks.hasEntitlement, +})); + +import { createReconciliationWorkload } from './reconciliationWorkload.js'; + +const settings = { + resyncConnectionIntervalMs: 24 * 60 * 60 * 1000, + reindexIntervalMs: 60 * 60 * 1000, + repoGarbageCollectionGracePeriodMs: 10 * 1000, + userDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, + repoDrivenPermissionSyncIntervalMs: 24 * 60 * 60 * 1000, +}; + +const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + flush: vi.fn(), +} satisfies JobLogger; + +describe('reconciliationWorkload', () => { + const connectionFindMany = vi.fn(); + const repoFindMany = vi.fn(); + const accountFindMany = vi.fn(); + const db = { + connection: { + findMany: connectionFindMany, + }, + repo: { + findMany: repoFindMany, + }, + account: { + findMany: accountFindMany, + }, + } as unknown as PrismaClient; + + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-07-27T12:00:00.000Z')); + connectionFindMany.mockReset().mockResolvedValue([]); + repoFindMany.mockReset().mockResolvedValue([]); + accountFindMany.mockReset().mockResolvedValue([]); + mocks.env.PERMISSION_SYNC_ENABLED = 'true'; + mocks.hasEntitlement.mockReset().mockResolvedValue(false); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test('runs every 10 seconds on the reconciliation queue', () => { + const workload = createReconciliationWorkload({ + db, + settings, + }); + + expect(workload.queueSpec.name).toBe('reconciliation'); + expect(workload.schedule).toEqual({ every: '10s' }); + expect(workload.concurrency).toBe(1); + }); + + test('triggers connection syncs for connections that are due', async () => { + connectionFindMany.mockResolvedValue([ + { id: 42, orgId: 1 }, + { id: 84, orgId: 2 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(connectionFindMany).toHaveBeenCalledWith({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + expect(trigger).toHaveBeenCalledTimes(2); + expect(trigger).toHaveBeenCalledWith('connection-sync', { + connectionId: 42, + orgId: 1, + }); + expect(trigger).toHaveBeenCalledWith('connection-sync', { + connectionId: 84, + orgId: 2, + }); + }); + + test('submits all due repos for indexing', async () => { + repoFindMany + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { id: 1 }, + { id: 2 }, + { id: 3 }, + { id: 4 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(trigger).toHaveBeenCalledTimes(4); + for (const repoId of [1, 2, 3, 4]) { + expect(trigger).toHaveBeenCalledWith('repo-index', { + repoId, + type: 'INDEX', + }); + } + }); + + test('schedules orphaned repos for cleanup', async () => { + repoFindMany + .mockResolvedValueOnce([{ id: 42 }]) + .mockResolvedValueOnce([]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(trigger).toHaveBeenCalledWith('repo-index', { + repoId: 42, + type: 'CLEANUP', + }); + }); + + test('submits due accounts for permission syncing when entitled', async () => { + mocks.hasEntitlement.mockResolvedValue(true); + accountFindMany.mockResolvedValue([ + { id: 'account_1' }, + { id: 'account_2' }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(accountFindMany).toHaveBeenCalledWith({ + where: { + AND: [ + { + providerType: { + in: ['github', 'gitlab', 'bitbucket-cloud', 'bitbucket-server'], + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + expect(trigger).toHaveBeenCalledWith('account-permission-sync', { + accountId: 'account_1', + }); + expect(trigger).toHaveBeenCalledWith('account-permission-sync', { + accountId: 'account_2', + }); + }); + + test('does not query accounts without the permission syncing entitlement', async () => { + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(accountFindMany).not.toHaveBeenCalled(); + expect(repoFindMany).toHaveBeenCalledTimes(2); + }); + + test('does not query accounts when permission syncing is disabled', async () => { + mocks.env.PERMISSION_SYNC_ENABLED = 'false'; + mocks.hasEntitlement.mockResolvedValue(true); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger: vi.fn(), + }); + + expect(mocks.hasEntitlement).not.toHaveBeenCalled(); + expect(accountFindMany).not.toHaveBeenCalled(); + expect(repoFindMany).toHaveBeenCalledTimes(2); + }); + + test('submits due private repos for permission syncing when entitled', async () => { + mocks.hasEntitlement.mockResolvedValue(true); + repoFindMany + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { id: 42 }, + { id: 84 }, + ]); + const trigger = vi.fn().mockResolvedValue('job-id'); + const workload = createReconciliationWorkload({ + db, + settings, + }); + + await workload.process({ + data: {}, + jobId: 'reconciliation-job', + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + logger, + updateProgress: vi.fn(), + trigger, + }); + + expect(repoFindMany).toHaveBeenNthCalledWith(3, { + where: { + AND: [ + { + isPublic: false, + }, + { + external_codeHostType: { + in: ['github', 'gitlab', 'bitbucketCloud', 'bitbucketServer'], + }, + }, + { + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: new Date('2026-07-26T12:00:00.000Z') } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { + repoId: 42, + }); + expect(trigger).toHaveBeenCalledWith('repo-permission-sync', { + repoId: 84, + }); + }); +}); diff --git a/packages/backend/src/reconciliationWorkload.ts b/packages/backend/src/reconciliationWorkload.ts new file mode 100644 index 000000000..fa2f29cfa --- /dev/null +++ b/packages/backend/src/reconciliationWorkload.ts @@ -0,0 +1,168 @@ +import { PrismaClient } from "@sourcebot/db"; +import { env, PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, RECONCILIATION_QUEUE } from "@sourcebot/shared"; +import { hasEntitlement } from "./entitlements.js"; +import { Settings, Workload } from "./types.js"; + +interface ReconciliationWorkloadDependencies { + db: PrismaClient; + settings: Settings; +} + +export const createReconciliationWorkload = ({ + db, + settings, +}: ReconciliationWorkloadDependencies): Workload<'reconciliation'> => ({ + concurrency: 1, + schedule: { every: '10s' }, + queueSpec: RECONCILIATION_QUEUE, + process: async ({ logger, trigger }) => { + // Connections + { + const connectionThreshold = new Date(Date.now() - settings.resyncConnectionIntervalMs); + const connections = await db.connection.findMany({ + where: { + OR: [ + { syncedAt: null }, + { syncedAt: { lt: connectionThreshold } }, + ], + }, + select: { + id: true, + orgId: true, + }, + }); + + await Promise.all(connections.map(async (connection) => { + await trigger('connection-sync', { + connectionId: connection.id, + orgId: connection.orgId, + }); + })); + } + + // Repo garbage collection + { + const cleanupThreshold = new Date(Date.now() - settings.repoGarbageCollectionGracePeriodMs); + const reposToCleanup = await db.repo.findMany({ + where: { + connections: { + none: {}, + }, + isAutoCleanupDisabled: false, + OR: [ + { indexedAt: null }, + { indexedAt: { lt: cleanupThreshold } }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(reposToCleanup.map(async ({ id }) => { + logger.debug(`Scheduling cleanup for repo ${id}`); + await trigger('repo-index', { + repoId: id, + type: 'CLEANUP', + }); + })); + } + + // Repo indexing + { + const indexThreshold = new Date(Date.now() - settings.reindexIntervalMs); + const reposToIndex = await db.repo.findMany({ + where: { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: indexThreshold } }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(reposToIndex.map(async ({ id }) => { + await trigger('repo-index', { + repoId: id, + type: 'INDEX', + }); + })); + } + + // Permission syncing + if ( + env.PERMISSION_SYNC_ENABLED === 'true' && + await hasEntitlement('permission-syncing') + ) { + const accountThreshold = new Date(Date.now() - settings.userDrivenPermissionSyncIntervalMs); + const accounts = await db.account.findMany({ + where: { + AND: [ + { + providerType: { + in: PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: accountThreshold } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(accounts.map(async ({ id }) => { + await trigger('account-permission-sync', { + accountId: id, + }); + })); + + const repoThreshold = new Date(Date.now() - settings.repoDrivenPermissionSyncIntervalMs); + const repos = await db.repo.findMany({ + where: { + AND: [ + { + isPublic: false, + }, + { + external_codeHostType: { + in: PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES, + }, + }, + { + connections: { + some: { + connection: { + enforcePermissions: true, + }, + }, + }, + }, + { + OR: [ + { permissionSyncedAt: null }, + { permissionSyncedAt: { lt: repoThreshold } }, + ], + }, + ], + }, + select: { + id: true, + }, + }); + + await Promise.all(repos.map(async ({ id }) => { + await trigger('repo-permission-sync', { + repoId: id, + }); + })); + } + }, +}); diff --git a/packages/backend/src/repoIndexManager.test.ts b/packages/backend/src/repoIndexManager.test.ts deleted file mode 100644 index 684f1a826..000000000 --- a/packages/backend/src/repoIndexManager.test.ts +++ /dev/null @@ -1,912 +0,0 @@ -import type { PrismaClient, Repo } from '@sourcebot/db'; -import { RepoIndexingJobStatus, RepoIndexingJobType } from '@sourcebot/db'; -import type { Job } from 'bullmq'; -import type { Redis } from 'ioredis'; -import { afterEach, beforeEach, describe, expect, Mock, test, vi } from 'vitest'; -import type { RepoWithConnections, Settings } from './types.js'; - -// Mock modules before importing the class under test -vi.mock('@sentry/node', () => ({ - captureException: vi.fn(), -})); - -vi.mock('@sourcebot/shared', () => ({ - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - env: { - DATA_CACHE_DIR: 'test-data', - REDIS_REMOVE_ON_COMPLETE: true, - REDIS_REMOVE_ON_FAIL: true, - }, - getRepoPath: vi.fn((repo: Repo) => ({ - path: `/test-data/repos/${repo.id}`, - isReadOnly: false, - })), - repoMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, - repoIndexingJobMetadataSchema: { - parse: vi.fn((metadata: unknown) => metadata ?? {}), - }, -})); - -vi.mock('./constants.js', () => ({ - WORKER_STOP_GRACEFUL_TIMEOUT_MS: 5000, - INDEX_CACHE_DIR: 'test-data/index', -})); - -vi.mock('./git.js', () => ({ - cloneRepository: vi.fn(), - fetchRepository: vi.fn(), - getBranches: vi.fn().mockResolvedValue([]), - getTags: vi.fn().mockResolvedValue([]), - getLocalDefaultBranch: vi.fn().mockResolvedValue('main'), - getCommitHashForRefName: vi.fn().mockResolvedValue('abc123'), - getLatestCommitTimestamp: vi.fn().mockResolvedValue(new Date()), - isPathAValidGitRepoRoot: vi.fn().mockResolvedValue(true), - isRepoEmpty: vi.fn().mockResolvedValue(false), - unsetGitConfig: vi.fn(), - upsertGitConfig: vi.fn(), - writeCommitGraph: vi.fn(), -})); - -vi.mock('./zoekt.js', () => ({ - indexGitRepository: vi.fn().mockResolvedValue({ stdout: '', stderr: '' }), -})); - -vi.mock('./posthog.js', () => ({ - captureEvent: vi.fn(), -})); - -vi.mock('./utils.js', () => ({ - getAuthCredentialsForRepo: vi.fn().mockResolvedValue(null), - getShardPrefix: vi.fn((orgId: number, repoId: number) => `${orgId}_${repoId}`), - measure: vi.fn(async (cb: () => Promise) => { - const data = await cb(); - return { data, durationMs: 100 }; - }), - setIntervalAsync: vi.fn((cb: () => void, _interval: number) => { - // Return a mock interval ID - return { unref: vi.fn() } as unknown as NodeJS.Timeout; - }), -})); - -vi.mock('fs', () => ({ - existsSync: vi.fn().mockReturnValue(false), -})); - -vi.mock('fs/promises', () => ({ - rm: vi.fn(), - readdir: vi.fn().mockResolvedValue([]), -})); - -// Mock BullMQ -const mockQueueAdd = vi.fn().mockResolvedValue(undefined); -const mockQueueClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerClose = vi.fn().mockResolvedValue(undefined); -const mockWorkerOn = vi.fn(); - -vi.mock('bullmq', () => ({ - Queue: vi.fn().mockImplementation(function () { - return { - add: mockQueueAdd, - close: mockQueueClose, - }; - }), - Worker: vi.fn().mockImplementation(function (_name: string, processor: unknown) { - return { - on: mockWorkerOn, - close: mockWorkerClose, - processJob: processor, - }; - }), - DelayedError: class DelayedError extends Error { - constructor(message: string) { - super(message); - this.name = 'DelayedError'; - } - }, -})); - -// Mock Redlock -const mockRedlockUsing = vi.fn(); -vi.mock('redlock', () => ({ - default: vi.fn().mockImplementation(function () { - return { - using: mockRedlockUsing, - }; - }), - ExecutionError: class ExecutionError extends Error { - constructor(message: string) { - super(message); - this.name = 'ExecutionError'; - } - }, -})); - -// Import after mocks are set up -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { ExecutionError } from 'redlock'; -import { - cloneRepository, - fetchRepository, - getBranches, - getTags, - isPathAValidGitRepoRoot, -} from './git.js'; -import { RepoIndexManager } from './repoIndexManager.js'; -import { indexGitRepository } from './zoekt.js'; - -// Helper to create mock Prisma client -const createMockPrisma = () => { - return { - repo: { - findMany: vi.fn().mockResolvedValue([]), - update: vi.fn(), - delete: vi.fn(), - }, - repoIndexingJob: { - createManyAndReturn: vi.fn().mockResolvedValue([]), - findUniqueOrThrow: vi.fn().mockResolvedValue({ status: RepoIndexingJobStatus.PENDING }), - update: vi.fn(), - }, - } as unknown as PrismaClient; -}; - -// Helper to create mock Redis -const createMockRedis = () => { - return {} as Redis; -}; - -// Helper to create mock Settings -const createMockSettings = (): Settings => ({ - maxFileSize: 2 * 1024 * 1024, - maxTrigramCount: 20000, - reindexIntervalMs: 1000 * 60 * 60, - resyncConnectionIntervalMs: 1000 * 60 * 60 * 24, - resyncConnectionPollingIntervalMs: 1000 * 1, - reindexRepoPollingIntervalMs: 1000 * 1, - maxConnectionSyncJobConcurrency: 8, - maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, - repoGarbageCollectionGracePeriodMs: 10 * 1000, - repoIndexTimeoutMs: 1000 * 60 * 60 * 2, - enablePublicAccess: false, - experiment_repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - experiment_userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - repoDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - userDrivenPermissionSyncIntervalMs: 1000 * 60 * 60 * 24, - maxAccountPermissionSyncJobConcurrency: 8, - maxRepoPermissionSyncJobConcurrency: 8, -}); - -// Helper to create mock PromClient -const createMockPromClient = () => ({ - pendingRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - activeRepoIndexJobs: { inc: vi.fn(), dec: vi.fn() }, - repoIndexJobSuccessTotal: { inc: vi.fn() }, - repoIndexJobFailTotal: { inc: vi.fn() }, -}); - -// Helper to create a mock repo -const createMockRepo = (overrides: Partial = {}): Repo => ({ - id: 1, - name: 'test-repo', - cloneUrl: 'https://github.com/test/repo.git', - orgId: 1, - indexedAt: null, - indexedCommitHash: null, - defaultBranch: 'main', - metadata: {}, - repoIndexingStatus: 'CREATED', - latestIndexingJobStatus: null, - latestConnectionSyncJobStatus: null, - external_id: 'test-external-id', - external_codeHostType: 'github', - external_codeHostUrl: 'https://github.com', - pushedAt: null, - createdAt: new Date(), - updatedAt: new Date(), - isFork: false, - isArchived: false, - isAutoCleanupDisabled: false, - ...overrides, -} as Repo); - -// Helper to create a mock repoWithConnections -const createMockRepoWithConnections = (overrides: Partial = {}): RepoWithConnections => ({ - ...createMockRepo(), - connections: [], - ...overrides, -}); - -describe('RepoIndexManager', () => { - let mockPrisma: PrismaClient; - let mockRedis: Redis; - let mockSettings: Settings; - let mockPromClient: ReturnType; - let manager: RepoIndexManager; - - beforeEach(() => { - vi.clearAllMocks(); - mockPrisma = createMockPrisma(); - mockRedis = createMockRedis(); - mockSettings = createMockSettings(); - mockPromClient = createMockPromClient(); - - // Default redlock behavior - execute the callback immediately - mockRedlockUsing.mockImplementation(async (_keys: string[], _ttl: number, cb: (signal: AbortSignal) => Promise) => { - const signal = new AbortController().signal; - return cb(signal); - }); - }); - - afterEach(async () => { - if (manager) { - await manager.dispose(); - } - }); - - describe('Job Processing - Success', () => { - test('clones new repository when directory does not exist', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // Set up mocks for job processing - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - // Simulate processing a job - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - // Get the worker processor callback - const { Worker } = await import('bullmq'); - const workerCalls = (Worker as unknown as Mock).mock.calls; - expect(workerCalls.length).toBeGreaterThan(0); - const processor = workerCalls[0][1]; - - // Execute the processor - await processor(mockJob); - - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('deletes directory and performs fresh clone when path exists but is not a valid git repo root', async () => { - const repo = createMockRepoWithConnections(); - // Path exists initially but after rm is called, it no longer exists - // First two calls return true (first check + check before delete), then false (after deletion) - (existsSync as Mock) - .mockReturnValueOnce(true) // First existsSync check - path exists - .mockReturnValueOnce(false); // Second existsSync check - path deleted, trigger clone - (isPathAValidGitRepoRoot as Mock).mockResolvedValue(false); - (cloneRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the invalid directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should perform a fresh clone after deletion - expect(cloneRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('fetches existing repository when directory exists', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - (fetchRepository as Mock).mockResolvedValue(undefined); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(fetchRepository).toHaveBeenCalledWith( - expect.objectContaining({ - cloneUrl: repo.cloneUrl, - path: expect.stringContaining(`${repo.id}`), - }) - ); - }); - - test('invokes zoekt-git-index with correct arguments', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - expect.arrayContaining(['refs/heads/main']), - expect.any(Object) - ); - }); - - test('keeps default branch and truncates to the first 63 matching tags', async () => { - const newestTagsFirst = Array.from( - { length: 70 }, - (_, index) => `v${70 - index}.0.0`, - ); - const repo = createMockRepoWithConnections({ - metadata: { - tags: ['**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getTags as Mock).mockResolvedValue(newestTagsFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - expect(indexGitRepository).toHaveBeenCalledWith( - repo, - mockSettings, - [ - 'refs/heads/main', - ...newestTagsFirst - .slice(0, 63) - .map((tag) => `refs/tags/${tag}`), - ], - expect.any(Object) - ); - }); - - test('de-duplicates the default branch before truncating matching branches', async () => { - const newestBranchesFirst = [ - 'feature/newest', - 'main', - ...Array.from( - { length: 68 }, - (_, index) => `feature/${68 - index}`, - ), - ]; - const repo = createMockRepoWithConnections({ - metadata: { - branches: ['main', 'feature/**'], - }, - }); - (existsSync as Mock).mockReturnValue(true); - (getBranches as Mock).mockResolvedValue(newestBranchesFirst); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - const revisions = (indexGitRepository as Mock).mock.calls.at(-1)?.[2] as string[]; - - expect(revisions).toHaveLength(64); - expect(revisions.filter((revision) => revision === 'refs/heads/main')).toHaveLength(1); - expect(revisions[0]).toBe('refs/heads/main'); - expect(revisions).toContain('refs/heads/feature/newest'); - expect(revisions).not.toContain('refs/heads/feature/6'); - }); - - test('updates repo.indexedAt and indexedCommitHash on completion', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - // The onJobCompleted handler reads the job via findUniqueOrThrow, then marks it - // COMPLETED and updates the repo (indexedAt, etc.) in a single repoIndexingJob.update - // with a nested repo update. - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - // Get the onCompleted handler - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - expect(onCompletedHandler).toBeDefined(); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - // The job status and indexedAt must be written together (single transaction) to - // close the race where the scheduler sees a completed job but a stale indexedAt. - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - completedAt: expect.any(Date), - repo: { - update: expect.objectContaining({ - indexedAt: expect.any(Date), - indexedCommitHash: 'abc123', - }), - }, - }), - }) - ); - }); - }); - - describe('Job Processing - Failure', () => { - test('marks job as FAILED when git clone throws', async () => { - const repo = createMockRepoWithConnections(); - const cloneError = new Error('Clone failed: authentication error'); - (existsSync as Mock).mockReturnValue(false); - (cloneRepository as Mock).mockRejectedValue(cloneError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - // Get the onFailed handler - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - expect(onFailedHandler).toBeDefined(); - - await onFailedHandler(mockJob, cloneError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: cloneError.message, - }), - }) - ); - - expect(mockPromClient.repoIndexJobFailTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'index', - }); - }); - - test('marks job as FAILED when zoekt-git-index fails', async () => { - const repo = createMockRepoWithConnections(); - const indexError = new Error('zoekt-git-index: failed to index'); - (existsSync as Mock).mockReturnValue(true); - (indexGitRepository as Mock).mockRejectedValue(indexError); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock) - .mockResolvedValueOnce({ type: RepoIndexingJobType.INDEX, repo }) - .mockResolvedValueOnce({ repo }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - await onFailedHandler(mockJob, indexError); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: indexError.message, - }), - }) - ); - }); - }); - - describe('Concurrency Control', () => { - test('prevents concurrent jobs for the same repo via redlock', async () => { - const repo = createMockRepoWithConnections(); - - // Simulate lock acquisition failure - mockRedlockUsing.mockRejectedValue(new ExecutionError('Lock already held')); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn().mockResolvedValue(undefined), - token: 'test-token', - } as unknown as Job; - - const { Worker, DelayedError } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - - // The processor should throw a DelayedError when lock cannot be acquired - await expect(processor(mockJob)).rejects.toThrow('locked'); - - // Verify moveToDelayed was called to retry later - expect(mockJob.moveToDelayed).toHaveBeenCalled(); - }); - }); - - describe('Cleanup Jobs', () => { - test('deletes repo directory and index shards', async () => { - const repo = createMockRepoWithConnections({ id: 5, orgId: 2 }); - (existsSync as Mock).mockReturnValue(true); - (readdir as Mock).mockResolvedValue(['2_5_v1.zoekt', '2_5_v2.zoekt', 'other_file.zoekt']); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repo, - }); - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Should delete the repo directory - expect(rm).toHaveBeenCalledWith( - expect.stringContaining(`${repo.id}`), - { recursive: true, force: true } - ); - - // Should delete shard files matching the prefix - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v1.zoekt'), - { force: true } - ); - expect(rm).toHaveBeenCalledWith( - expect.stringContaining('2_5_v2.zoekt'), - { force: true } - ); - }); - - test('removes repo from database after cleanup', async () => { - const repo = createMockRepoWithConnections({ id: 3 }); - (existsSync as Mock).mockReturnValue(false); - (readdir as Mock).mockResolvedValue([]); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.CLEANUP, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - (mockPrisma.repo.delete as Mock).mockResolvedValue(repo); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'cleanup-job-1', - type: 'CLEANUP', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repo.delete).toHaveBeenCalledWith({ - where: { id: repo.id }, - }); - - expect(mockPromClient.repoIndexJobSuccessTotal.inc).toHaveBeenCalledWith({ - repo: repo.name, - type: 'cleanup', - }); - }); - }); - - describe('latestIndexingJobStatus Updates', () => { - test('sets latestIndexingJobStatus to IN_PROGRESS when job starts', async () => { - const repo = createMockRepoWithConnections(); - (existsSync as Mock).mockReturnValue(true); - // Ensure indexGitRepository resolves for this test - (indexGitRepository as Mock).mockResolvedValue({ stdout: '', stderr: '' }); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - status: RepoIndexingJobStatus.PENDING, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repo, - }); - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - moveToDelayed: vi.fn(), - } as unknown as Job; - - const { Worker } = await import('bullmq'); - const processor = (Worker as unknown as Mock).mock.calls[0][1]; - await processor(mockJob); - - // Verify the first update call sets latestIndexingJobStatus to IN_PROGRESS - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - }, - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to COMPLETED when job succeeds', async () => { - const repo = createMockRepoWithConnections(); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.findUniqueOrThrow as Mock).mockResolvedValue({ - type: RepoIndexingJobType.INDEX, - repoId: repo.id, - repo, - metadata: {}, - }); - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onCompletedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'completed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - } as unknown as Job; - - await onCompletedHandler(mockJob); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.COMPLETED, - repo: { - update: expect.objectContaining({ - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - }), - }, - }), - }) - ); - }); - - test('sets latestIndexingJobStatus to FAILED when job fails', async () => { - const repo = createMockRepoWithConnections(); - const error = new Error('Job processing failed'); - - manager = new RepoIndexManager(mockPrisma, mockSettings, mockRedis, mockPromClient as any); - - (mockPrisma.repoIndexingJob.update as Mock).mockResolvedValue({ repo }); - - const onFailedHandler = mockWorkerOn.mock.calls.find((call: unknown[]) => call[0] === 'failed')?.[1]; - - const mockJob = { - data: { - jobId: 'job-1', - type: 'INDEX', - repoId: repo.id, - repoName: repo.name, - }, - getState: vi.fn().mockResolvedValue('failed'), - } as unknown as Job; - - await onFailedHandler(mockJob, error); - - expect(mockPrisma.repoIndexingJob.update).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: 'job-1' }, - data: expect.objectContaining({ - status: RepoIndexingJobStatus.FAILED, - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - }, - }, - }), - }) - ); - }); - }); -}); diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts deleted file mode 100644 index aea1291dc..000000000 --- a/packages/backend/src/repoIndexManager.ts +++ /dev/null @@ -1,769 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; -import { createLogger, env, getRepoPath, Logger, getRepoIdFromPath, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared"; -import { DelayedError, Job, Queue, Worker } from "bullmq"; -import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; -import { Redis } from 'ioredis'; -import micromatch from 'micromatch'; -import Redlock, { ExecutionError } from 'redlock'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js'; -import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; -import { captureEvent } from './posthog.js'; -import { PromClient } from './promClient.js'; -import { RepoWithConnections, Settings } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure, setIntervalAsync } from './utils.js'; -import { cleanupTempShards, indexGitRepository } from './zoekt.js'; - -const LOG_TAG = 'repo-index-manager'; -const logger = createLogger(LOG_TAG); -const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -const QUEUE_NAME = 'repo-index-queue'; - -type JobPayload = { - type: 'INDEX' | 'CLEANUP'; - jobId: string; - repoId: number; - repoName: string; -}; - -// Lock TTL with auto-extension - minimizes dead lock time after crashes -const LOCK_TTL_MS = 60 * 1000; // 1 minute -const LOCK_PREFIX = `bullmq:${QUEUE_NAME}:lock:`; - -// Delay before retrying a job when the group lock cannot be acquired -const LOCK_RETRY_DELAY_MS = 5000; - -/** - * Manages the lifecycle of repository data on disk, including git working copies - * and search index shards. Handles both indexing operations (cloning/fetching repos - * and building search indexes) and cleanup operations (removing orphaned repos and - * their associated data). - * - * Uses a job queue system to process indexing and cleanup tasks asynchronously, - * with configurable concurrency limits and retry logic. Automatically schedules - * re-indexing of repos based on configured intervals and manages garbage collection - * of repos that are no longer connected to any source. - */ -export class RepoIndexManager { - private interval?: NodeJS.Timeout; - private queue: Queue; - private worker: Worker; - private redlock: Redlock; - private abortController: AbortController; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - this.abortController = new AbortController(); - - this.queue = new Queue(QUEUE_NAME, { - connection: redis, - defaultJobOptions: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - attempts: 2, - }, - }); - - this.redlock = new Redlock([redis], { - retryCount: 0, // Don't retry - we'll delay the job instead - automaticExtensionThreshold: LOCK_TTL_MS / 2, // Extend when 50% of TTL remains - }); - - this.worker = new Worker( - QUEUE_NAME, - this.processJob.bind(this), - { - connection: redis, - concurrency: this.settings.maxRepoIndexingJobConcurrency, - maxStalledCount: 1, - } - ); - - this.worker.on('completed', this.onJobCompleted.bind(this)); - this.worker.on('failed', this.onJobMaybeFailed.bind(this)); - this.worker.on('stalled', (jobId) => { - // Just log - BullMQ will automatically retry the job (up to maxStalledCount times). - // If all retries fail, onJobMaybeFailed will handle marking it as failed. - logger.warn(`Job ${jobId} stalled - BullMQ will retry`); - }); - this.worker.on('error', (error) => { - logger.error(`Index syncer worker error:`, error); - }); - } - - public async startScheduler() { - logger.debug('Starting scheduler'); - // Cleanup any orphaned disk resources on startup - await this.cleanupOrphanedDiskResources(); - this.interval = setIntervalAsync(async () => { - await this.scheduleIndexJobs(); - await this.scheduleCleanupJobs(); - }, this.settings.reindexRepoPollingIntervalMs); - } - - private async scheduleIndexJobs() { - const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToIndex = await this.db.repo.findMany({ - where: { - AND: [ - { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.INDEX, - }, - { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: RepoIndexingJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } - ] - } - ] - } - } - } - } - ], - }, - }); - - if (reposToIndex.length > 0) { - await this.createJobs(reposToIndex, RepoIndexingJobType.INDEX); - } - } - - private async scheduleCleanupJobs() { - const gcGracePeriodMs = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); - const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs); - - const reposToCleanup = await this.db.repo.findMany({ - where: { - connections: { - none: {} - }, - isAutoCleanupDisabled: false, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: gcGracePeriodMs } }, - ], - NOT: { - jobs: { - some: { - AND: [ - { - type: RepoIndexingJobType.CLEANUP, - }, - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - }, - }, - { - createdAt: { - gt: timeoutDate, - } - } - ] - } - } - } - } - }); - - if (reposToCleanup.length > 0) { - await this.createJobs(reposToCleanup, RepoIndexingJobType.CLEANUP); - } - } - - public async createJobs(repos: Repo[], type: RepoIndexingJobType) { - // @note: we don't perform this in a transaction because - // we want to avoid the situation where a job is created and run - // prior to the transaction being committed. - const jobs = await this.db.repoIndexingJob.createManyAndReturn({ - data: repos.map(repo => ({ - type, - repoId: repo.id, - })), - include: { - repo: true, - } - }); - - for (const job of jobs) { - await this.queue.add( - 'repo-index-job', - { - jobId: job.id, - type, - repoName: job.repo.name, - repoId: job.repo.id, - }, - { jobId: job.id } - ); - - const jobTypeLabel = getJobTypePrometheusLabel(type); - this.promClient.pendingRepoIndexJobs.inc({ repo: job.repo.name, type: jobTypeLabel }); - } - - return jobs.map(job => job.id); - } - - private async processJob(job: Job): Promise { - const groupId = `repo:${job.data.repoId}`; - const lockKey = `${LOCK_PREFIX}${groupId}`; - - try { - return await this.redlock.using([lockKey], LOCK_TTL_MS, async (lockSignal: AbortSignal) => { - const signal = AbortSignal.any([ - this.abortController.signal, - lockSignal, - ]); - - return await this.runJob(job, signal); - }); - } catch (error) { - if (error instanceof ExecutionError) { - // Lock could not be acquired - another job for this group is running - // Delay this job and let BullMQ retry later - // DelayedError tells BullMQ to delay without counting as a failed attempt - logger.debug(`Group ${groupId} locked, delaying job ${job.id}`); - await job.moveToDelayed(Date.now() + LOCK_RETRY_DELAY_MS, job.token); - throw new DelayedError(`Group ${groupId} locked, delaying job`); - } - throw error; - } - } - - private async runJob(job: Job, signal: AbortSignal) { - const id = job.data.jobId; - const logger = createJobLogger(id); - logger.debug(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId})`); - - const currentStatus = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { - id, - }, - select: { - status: true, - } - }); - - // Fail safe: if the job is not PENDING (first run) or IN_PROGRESS (retry), it indicates the job - // is in an invalid state and should be skipped. - if ( - currentStatus.status !== RepoIndexingJobStatus.PENDING && - currentStatus.status !== RepoIndexingJobStatus.IN_PROGRESS - ) { - throw new Error(`Job ${id} is not in a valid state. Expected: ${RepoIndexingJobStatus.PENDING} or ${RepoIndexingJobStatus.IN_PROGRESS}. Actual: ${currentStatus.status}. Skipping.`); - } - - const { repo, type: jobType } = await this.db.repoIndexingJob.update({ - where: { - id, - }, - data: { - status: RepoIndexingJobStatus.IN_PROGRESS, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - } - } - }, - select: { - type: true, - repo: { - include: { - connections: { - include: { - connection: true, - } - } - } - } - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobType); - this.promClient.pendingRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.activeRepoIndexJobs.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobType === RepoIndexingJobType.INDEX) { - const revisions = await this.indexRepository(repo, logger, signal); - - await this.db.repoIndexingJob.update({ - where: { id }, - data: { - metadata: { - indexedRevisions: revisions, - } satisfies RepoIndexingJobMetadata, - }, - }); - } else if (jobType === RepoIndexingJobType.CLEANUP) { - await this.cleanupRepository(repo, logger); - } - - } - - private async indexRepository(repo: RepoWithConnections, logger: Logger, signal: AbortSignal) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - - const metadata = repoMetadataSchema.parse(repo.metadata); - - const credentials = await getAuthCredentialsForRepo(repo, logger); - const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; - const authHeader = credentials?.authHeader ?? undefined; - - // If the repo path exists but it is not a valid git repository root, this indicates - // that the repository is in a bad state. To fix, we remove the directory and perform - // a fresh clone. - if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { - const isValidGitRepo = await isPathAValidGitRepoRoot({ - path: repoPath, - signal, - }); - - if (!isValidGitRepo && !isReadOnly) { - logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); - await rm(repoPath, { recursive: true, force: true }); - } - } - - if (existsSync(repoPath) && !isReadOnly) { - // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ - // write the clone URL (which could contain a auth token) to the - // `remote.origin.url` entry. For the upgrade scenario, we want - // to unset this key since it is no longer needed, hence this line. - // This will no-op if the key is already unset. - // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 - await unsetGitConfig({ - path: repoPath, - keys: ["remote.origin.url"], - signal, - }); - - logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); - const { durationMs } = await measure(() => fetchRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal, - })); - const fetchDuration_s = durationMs / 1000; - - logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); - - // Update the commit-graph after fetch. Force a full backfill the first time we - // see this repo after the --changed-paths rollout, so historical commits get - // Bloom filters. Subsequent fetches do a cheap incremental write. - const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; - if (needsBackfill) { - logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); - } - await writeCommitGraph({ - path: repoPath, - forceBackfill: needsBackfill, - signal, - }); - } else if (!isReadOnly) { - logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); - - const { durationMs } = await measure(() => cloneRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - }, - signal - })); - const cloneDuration_s = durationMs / 1000; - - logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); - - // Write the commit-graph for the freshly cloned repo. - await writeCommitGraph({ - path: repoPath, - signal, - }); - } - - // Record that this repo's commit-graph now includes changed-path Bloom filters - // for its full history (either freshly written during clone, or backfilled above - // during fetch). - if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { - await this.db.repo.update({ - where: { id: repo.id }, - data: { - metadata: { - ...metadata, - commitGraphChangedPathsBackfilledAt: new Date().toISOString(), - } satisfies RepoMetadata, - }, - }); - } - - // Regardless of clone or fetch, always upsert the git config for the repo. - // This ensures that the git config is always up to date for whatever we - // have in the DB. - if (metadata.gitConfig && !isReadOnly) { - await upsertGitConfig({ - path: repoPath, - gitConfig: metadata.gitConfig, - signal, - }); - } - - const defaultBranch = await getLocalDefaultBranch({ - path: repoPath, - }); - - // Ensure defaultBranch has refs/heads/ prefix for consistent searching - const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') - ? `refs/heads/${defaultBranch}` - : defaultBranch; - - let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; - - if (metadata.branches) { - const branchGlobs = metadata.branches - const allBranches = await getBranches(repoPath); - const matchingBranches = - allBranches - .filter((branch) => micromatch.isMatch(branch, branchGlobs)) - .map((branch) => `refs/heads/${branch}`); - - revisions = [ - ...revisions, - ...matchingBranches - ]; - } - - if (metadata.tags) { - const tagGlobs = metadata.tags; - const allTags = await getTags(repoPath); - const matchingTags = - allTags - .filter((tag) => micromatch.isMatch(tag, tagGlobs)) - .map((tag) => `refs/tags/${tag}`); - - revisions = [ - ...revisions, - ...matchingTags - ]; - } - - // De-duplicate revisions to ensure we don't have duplicate branches/tags - revisions = [...new Set(revisions)]; - - // zoekt has a limit of 64 branches/tags to index. - if (revisions.length > 64) { - logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); - captureEvent('backend_revisions_truncated', { - repoId: repo.id, - revisionCount: revisions.length, - }); - revisions = revisions.slice(0, 64); - } - - logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); - try { - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, revisions, signal)); - const indexDuration_s = durationMs / 1000; - logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); - } catch (error) { - // Clean up any temporary shard files left behind by the failed indexing operation. - // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. - logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); - await cleanupTempShards(repo); - throw error; - } - - return revisions; - } - - private async cleanupRepository(repo: Repo, logger: Logger) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - if (existsSync(repoPath) && !isReadOnly) { - logger.debug(`Deleting repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); - for (const file of files) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting shard file ${filePath}`); - await rm(filePath, { force: true }); - } - } - - private async onJobCompleted(job: Job) { - try { - const logger = createJobLogger(job.data.jobId); - const jobData = await this.db.repoIndexingJob.findUniqueOrThrow({ - where: { id: job.data.jobId }, - include: { - repo: true, - } - }); - - const jobTypeLabel = getJobTypePrometheusLabel(jobData.type); - // @note: capture this before the update below, since the update sets indexedAt. - const isFirstIndex = jobData.repo.indexedAt === null; - - if (jobData.type === RepoIndexingJobType.INDEX) { - const { path: repoPath } = getRepoPath(jobData.repo); - const isEmpty = await isRepoEmpty({ path: repoPath }); - const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ - path: repoPath, - refName: 'HEAD', - }); - - const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); - const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); - - const jobMetadata = repoIndexingJobMetadataSchema.parse(jobData.metadata); - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - indexedAt: new Date(), - indexedCommitHash: commitHash, - pushedAt: pushedAt, - metadata: { - ...(jobData.repo.metadata as RepoMetadata), - indexedRevisions: jobMetadata.indexedRevisions, - } satisfies RepoMetadata, - // @note: always update the default branch. While this field can be set - // during connection syncing, by setting it here we ensure that a) the - // default branch is as up to date as possible (since repo indexing happens - // more frequently than connection syncing) and b) for hosts where it is - // impossible to determine the default branch from the host's API - // (e.g., generic git url), we still set the default branch here. - defaultBranch: defaultBranch, - } - } - }, - include: { - repo: true, - } - }); - - logger.debug(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - else if (jobData.type === RepoIndexingJobType.CLEANUP) { - await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - } - }); - - const repo = await this.db.repo.delete({ - where: { id: jobData.repoId }, - }); - - logger.debug(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); - } - - // Track metrics for successful job - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobSuccessTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - if (jobData.type === RepoIndexingJobType.INDEX && isFirstIndex) { - captureEvent('backend_repo_first_indexed', { - repoId: job.data.repoId, - type: jobData.repo.external_codeHostType, - }); - } - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`onJobCompleted\`.`, error); - } - } - - private async onJobMaybeFailed(job: Job | undefined, error: Error) { - try { - if (!job) { - logger.error(`Job failed but job object is undefined. Error: ${error.message}`); - return; - } - - const jobLogger = createJobLogger(job.data.jobId); - const jobTypeLabel = getJobTypePrometheusLabel(job.data.type); - - // @note: we need to check the job state to determine if the job failed, - // or if it is being retried. - const jobState = await job.getState(); - if (jobState !== 'failed') { - jobLogger.warn(`Job ${job.id} for repo ${job.data.repoName} (id: ${job.data.repoId}) failed. Retrying... Reason: ${error.message}`); - return; - } - - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - repo: { - update: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - } - } - }, - select: { repo: true } - }); - - this.promClient.activeRepoIndexJobs.dec({ repo: job.data.repoName, type: jobTypeLabel }); - this.promClient.repoIndexJobFailTotal.inc({ repo: job.data.repoName, type: jobTypeLabel }); - - jobLogger.error(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Reason: ${error.message}`); - - captureEvent('backend_repo_index_job_failed', { - repoId: job.data.repoId, - jobType: job.data.type, - type: repo.external_codeHostType, - }); - } catch (err) { - Sentry.captureException(err); - logger.error(`Exception thrown while executing lifecycle function \`onJobMaybeFailed\`.`, err); - } - } - - // Scans the repos and index directories on disk and removes any entries - // that have no corresponding Repo record in the database. This handles - // edge cases where the DB and disk resources are out of sync. - private async cleanupOrphanedDiskResources() { - // --- Repo directories --- - // Dirs are named by repoId: DATA_CACHE_DIR/repos// - if (existsSync(REPOS_CACHE_DIR)) { - const entries = await readdir(REPOS_CACHE_DIR); - const repoIdToPath = new Map(); - for (const entry of entries) { - const repoPath = `${REPOS_CACHE_DIR}/${entry}`; - const repoId = getRepoIdFromPath(repoPath); - if (repoId !== undefined) { - repoIdToPath.set(repoId, repoPath); - } - } - - if (repoIdToPath.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToPath.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, repoPath] of repoIdToPath) { - if (!existingIds.has(repoId)) { - logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - } - } - } - - // --- Index shards --- - // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt - if (existsSync(INDEX_CACHE_DIR)) { - const entries = await readdir(INDEX_CACHE_DIR); - const repoIdToShards = new Map(); - for (const entry of entries) { - const repoId = getRepoIdFromShardFileName(entry); - if (repoId !== undefined) { - const shards = repoIdToShards.get(repoId) ?? []; - shards.push(entry); - repoIdToShards.set(repoId, shards); - } - } - - if (repoIdToShards.size > 0) { - const existingRepos = await this.db.repo.findMany({ - where: { id: { in: [...repoIdToShards.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, shards] of repoIdToShards) { - if (!existingIds.has(repoId)) { - for (const entry of shards) { - const shardPath = `${INDEX_CACHE_DIR}/${entry}`; - logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); - await rm(shardPath, { force: true }); - } - } - } - } - } - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - - // Signal all active jobs to abort - this.abortController.abort(); - - // Wait for worker to finish with timeout - await Promise.race([ - this.worker.close(), - new Promise(resolve => setTimeout(resolve, WORKER_STOP_GRACEFUL_TIMEOUT_MS)) - ]); - - // Locks will auto-expire via TTL, no need to manually release them - await this.queue.close(); - } -} - -const getJobTypePrometheusLabel = (type: RepoIndexingJobType) => type === RepoIndexingJobType.INDEX ? 'index' : 'cleanup'; diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts new file mode 100644 index 000000000..023a71e07 --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -0,0 +1,153 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; + +const repoIndexingJobUpsert = vi.fn(); +const repoIndexingJobUpdate = vi.fn(); +const repoUpdate = vi.fn(); +const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => + callback({ + repoIndexingJob: { + upsert: repoIndexingJobUpsert, + update: repoIndexingJobUpdate, + }, + repo: { + update: repoUpdate, + }, + }), +); + +const db = { + $transaction: transaction, +} as unknown as PrismaClient; + +const workload = createRepoIndexWorkload({ + db, + settings: { + maxRepoIndexingJobConcurrency: 2, + } as never, +}); + +const lifecycleLogger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +}; + +const lifecycleContext = { + data: { + repoId: 42, + type: "INDEX" as const, + }, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + logger: lifecycleLogger, +}; + +describe("repoIndexWorkload lifecycle", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test("declares database-backed lifecycle hooks", () => { + expect(workload.onStarted).toBeTypeOf("function"); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("marks the repo indexing job and repo as in progress when started", async () => { + await workload.onStarted?.(lifecycleContext); + + expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + update: { + status: "IN_PROGRESS", + completedAt: null, + errorMessage: null, + }, + create: { + id: "job-1", + repoId: 42, + type: "INDEX", + status: "IN_PROGRESS", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: "IN_PROGRESS", + }, + }); + }); + + test("marks the repo indexing job and repo as completed", async () => { + await workload.onCompleted?.(lifecycleContext, undefined); + + expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "COMPLETED", + completedAt: expect.any(Date), + errorMessage: null, + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: "COMPLETED", + }, + }); + }); + + test("does not update a completed cleanup job after its repo cascades the job row", async () => { + await workload.onCompleted?.( + { + ...lifecycleContext, + data: { + repoId: 42, + type: "CLEANUP", + }, + }, + undefined, + ); + + expect(transaction).not.toHaveBeenCalled(); + }); + + test("marks the repo indexing job and repo as failed after terminal failure", async () => { + await workload.onTerminalFailure?.( + lifecycleContext, + new Error("Unable to clone repository"), + ); + + expect(repoIndexingJobUpdate).toHaveBeenCalledWith({ + where: { + id: "job-1", + }, + data: { + status: "FAILED", + completedAt: expect.any(Date), + errorMessage: "Unable to clone repository", + }, + }); + expect(repoUpdate).toHaveBeenCalledWith({ + where: { + id: 42, + }, + data: { + latestIndexingJobStatus: "FAILED", + }, + }); + }); +}); diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts new file mode 100644 index 000000000..7e21d9493 --- /dev/null +++ b/packages/backend/src/repoIndexWorkload.ts @@ -0,0 +1,429 @@ +import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { createLogger, getRepoPath, JobLogSink, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { existsSync } from 'fs'; +import { readdir, rm } from 'fs/promises'; +import micromatch from 'micromatch'; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; +import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; +import { captureEvent } from './posthog.js'; +import { RepoWithConnections, Settings, Workload } from "./types.js"; +import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure } from './utils.js'; +import { cleanupTempShards, indexGitRepository } from './zoekt.js'; + +const LOG_TAG = 'repo-index-workload'; +const logger = createLogger(LOG_TAG); + +interface Props { + db: PrismaClient; + settings: Settings; +} + +export const createRepoIndexWorkload = ({ + db, + settings, +}: Props): Workload<'repo-index'> => ({ + queueSpec: REPO_INDEX_QUEUE, + concurrency: settings.maxRepoIndexingJobConcurrency, + process: async ({ data, logger: jobLogger, signal }) => { + const repo = await db.repo.findUniqueOrThrow({ + where: { id: data.repoId }, + include: { + connections: { + include: { + connection: true, + }, + }, + }, + }); + + jobLogger.debug(`Running ${data.type} job for repo ${repo.name} (id: ${repo.id})`); + + if (data.type === 'CLEANUP') { + await cleanupRepository(repo, jobLogger); + await db.repo.delete({ + where: { id: repo.id }, + }); + } else { + const isFirstIndex = repo.indexedAt === null; + const revisions = await indexRepository(db, settings, repo, jobLogger, signal); + const { path: repoPath } = getRepoPath(repo); + const isEmpty = await isRepoEmpty({ path: repoPath }); + const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ + path: repoPath, + refName: 'HEAD', + }); + const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); + const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); + const currentRepo = await db.repo.findUniqueOrThrow({ + where: { id: repo.id }, + select: { metadata: true }, + }); + + await db.repo.update({ + where: { id: repo.id }, + data: { + indexedAt: new Date(), + indexedCommitHash: commitHash, + pushedAt, + metadata: { + ...(currentRepo.metadata as RepoMetadata), + indexedRevisions: revisions, + } satisfies RepoMetadata, + defaultBranch, + }, + }); + + if (isFirstIndex) { + captureEvent('backend_repo_first_indexed', { + repoId: repo.id, + type: repo.external_codeHostType, + }); + } + } + }, + onStarted: async ({ data: { repoId, type }, jobId }) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.upsert({ + where: { + id: jobId, + }, + update: { + status: RepoIndexingJobStatus.IN_PROGRESS, + completedAt: null, + errorMessage: null, + }, + create: { + id: jobId, + repoId, + type: RepoIndexingJobType[type], + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, + }, + }); + }); + }, + onCompleted: async ({ data: { repoId, type }, jobId }) => { + // A successful cleanup deletes the Repo in `process`, which cascades to its + // RepoIndexingJob records. There is no row left to mark as completed. + if (type === RepoIndexingJobType.CLEANUP) { + return; + } + + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.COMPLETED, + completedAt: new Date(), + errorMessage: null, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, + }, + }); + }); + }, + onTerminalFailure: async ({ data: { repoId }, jobId }, error) => { + await db.$transaction(async (tx) => { + await tx.repoIndexingJob.update({ + where: { + id: jobId, + }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: error.message, + }, + }); + await tx.repo.update({ + where: { + id: repoId, + }, + data: { + latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, + }, + }); + }); + }, +}); + +const indexRepository = async ( + db: PrismaClient, + settings: Settings, + repo: RepoWithConnections, + logger: JobLogSink, + signal: AbortSignal, +) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + + const metadata = repoMetadataSchema.parse(repo.metadata); + + const credentials = await getAuthCredentialsForRepo(repo, logger); + const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; + const authHeader = credentials?.authHeader ?? undefined; + + // If the repo path exists but it is not a valid git repository root, this indicates + // that the repository is in a bad state. To fix, we remove the directory and perform + // a fresh clone. + if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot({ path: repoPath }))) { + const isValidGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + signal, + }); + + if (!isValidGitRepo && !isReadOnly) { + logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); + await rm(repoPath, { recursive: true, force: true }); + } + } + + if (existsSync(repoPath) && !isReadOnly) { + // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ + // write the clone URL (which could contain a auth token) to the + // `remote.origin.url` entry. For the upgrade scenario, we want + // to unset this key since it is no longer needed, hence this line. + // This will no-op if the key is already unset. + // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 + await unsetGitConfig({ + path: repoPath, + keys: ["remote.origin.url"], + signal, + }); + + logger.debug(`Fetching ${repo.name} (id: ${repo.id})...`); + const { durationMs } = await measure(() => fetchRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal, + })); + const fetchDuration_s = durationMs / 1000; + + logger.debug(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); + + // Update the commit-graph after fetch. Force a full backfill the first time we + // see this repo after the --changed-paths rollout, so historical commits get + // Bloom filters. Subsequent fetches do a cheap incremental write. + const needsBackfill = !metadata.commitGraphChangedPathsBackfilledAt; + if (needsBackfill) { + logger.debug(`Backfilling changed-path Bloom filters for ${repo.name} (id: ${repo.id})...`); + } + await writeCommitGraph({ + path: repoPath, + forceBackfill: needsBackfill, + signal, + }); + } else if (!isReadOnly) { + logger.debug(`Cloning ${repo.name} (id: ${repo.id})...`); + + const { durationMs } = await measure(() => cloneRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal + })); + const cloneDuration_s = durationMs / 1000; + + logger.debug(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); + + // Write the commit-graph for the freshly cloned repo. + await writeCommitGraph({ + path: repoPath, + signal, + }); + } + + // Record that this repo's commit-graph now includes changed-path Bloom filters + // for its full history (either freshly written during clone, or backfilled above + // during fetch). + if (!isReadOnly && !metadata.commitGraphChangedPathsBackfilledAt) { + await db.repo.update({ + where: { id: repo.id }, + data: { + metadata: { + ...metadata, + commitGraphChangedPathsBackfilledAt: new Date().toISOString(), + } satisfies RepoMetadata, + }, + }); + } + + // Regardless of clone or fetch, always upsert the git config for the repo. + // This ensures that the git config is always up to date for whatever we + // have in the DB. + if (metadata.gitConfig && !isReadOnly) { + await upsertGitConfig({ + path: repoPath, + gitConfig: metadata.gitConfig, + signal, + }); + } + + const defaultBranch = await getLocalDefaultBranch({ + path: repoPath, + }); + + // Ensure defaultBranch has refs/heads/ prefix for consistent searching + const defaultBranchWithPrefix = defaultBranch && !defaultBranch.startsWith('refs/') + ? `refs/heads/${defaultBranch}` + : defaultBranch; + + let revisions = defaultBranchWithPrefix ? [defaultBranchWithPrefix] : ['HEAD']; + + if (metadata.branches) { + const branchGlobs = metadata.branches + const allBranches = await getBranches(repoPath); + const matchingBranches = + allBranches + .filter((branch) => micromatch.isMatch(branch, branchGlobs)) + .map((branch) => `refs/heads/${branch}`); + + revisions = [ + ...revisions, + ...matchingBranches + ]; + } + + if (metadata.tags) { + const tagGlobs = metadata.tags; + const allTags = await getTags(repoPath); + const matchingTags = + allTags + .filter((tag) => micromatch.isMatch(tag, tagGlobs)) + .map((tag) => `refs/tags/${tag}`); + + revisions = [ + ...revisions, + ...matchingTags + ]; + } + + // De-duplicate revisions to ensure we don't have duplicate branches/tags + revisions = [...new Set(revisions)]; + + // zoekt has a limit of 64 branches/tags to index. + if (revisions.length > 64) { + logger.warn(`Too many revisions (${revisions.length}) for repo ${repo.id}, truncating to 64`); + captureEvent('backend_revisions_truncated', { + repoId: repo.id, + revisionCount: revisions.length, + }); + revisions = revisions.slice(0, 64); + } + + logger.debug(`Indexing ${repo.name} (id: ${repo.id})...`); + try { + const { durationMs } = await measure(() => indexGitRepository(repo, settings, revisions, signal)); + const indexDuration_s = durationMs / 1000; + logger.debug(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); + } catch (error) { + // Clean up any temporary shard files left behind by the failed indexing operation. + // Zoekt creates .tmp files during indexing which can accumulate if indexing fails repeatedly. + logger.warn(`Indexing failed for ${repo.name} (id: ${repo.id}), cleaning up temp shard files...`); + await cleanupTempShards(repo); + throw error; + } + + return revisions; +}; + +const cleanupRepository = async (repo: Repo, logger: JobLogSink) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + if (existsSync(repoPath) && !isReadOnly) { + logger.debug(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const shardPrefix = getShardPrefix(repo.orgId, repo.id); + const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); + for (const file of files) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +// Scans the repos and index directories on disk and removes any entries +// that have no corresponding Repo record in the database. This handles +// edge cases where the DB and disk resources are out of sync. +export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { + // --- Repo directories --- + // Dirs are named by repoId: DATA_CACHE_DIR/repos// + if (existsSync(REPOS_CACHE_DIR)) { + const entries = await readdir(REPOS_CACHE_DIR); + const repoIdToPath = new Map(); + for (const entry of entries) { + const repoPath = `${REPOS_CACHE_DIR}/${entry}`; + const repoId = getRepoIdFromPath(repoPath); + if (repoId !== undefined) { + repoIdToPath.set(repoId, repoPath); + } + } + + if (repoIdToPath.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToPath.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, repoPath] of repoIdToPath) { + if (!existingIds.has(repoId)) { + logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + } + } + } + + // --- Index shards --- + // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt + if (existsSync(INDEX_CACHE_DIR)) { + const entries = await readdir(INDEX_CACHE_DIR); + const repoIdToShards = new Map(); + for (const entry of entries) { + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + const shards = repoIdToShards.get(repoId) ?? []; + shards.push(entry); + repoIdToShards.set(repoId, shards); + } + } + + if (repoIdToShards.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToShards.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map(r => r.id)); + for (const [repoId, shards] of repoIdToShards) { + if (!existingIds.has(repoId)) { + for (const entry of shards) { + const shardPath = `${INDEX_CACHE_DIR}/${entry}`; + logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); + await rm(shardPath, { force: true }); + } + } + } + } + } +}; diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 8803b48b9..1cc29e147 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -1,27 +1,133 @@ -import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; +import { + Connection, + PrismaClient, + Repo, + RepoToConnection, +} from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; +import { DataOf, JobLogSink, QueueName, QueueSpec } from "@sourcebot/shared"; export type Settings = Required; // @see : https://stackoverflow.com/a/61132308 -export type DeepPartial = T extends object ? { - [P in keyof T]?: DeepPartial; -} : T; +export type DeepPartial = T extends object + ? { + [P in keyof T]?: DeepPartial; + } + : T; // @see: https://stackoverflow.com/a/69328045 export type WithRequired = T & { [P in K]-?: T[P] }; -export type RepoWithConnections = Repo & { connections: (RepoToConnection & { connection: Connection })[] }; - +export type RepoWithConnections = Repo & { + connections: (RepoToConnection & { connection: Connection })[]; +}; export type RepoAuthCredentials = { hostUrl?: string; token: string; cloneUrlWithToken?: string; authHeader?: string; - /** The connection that configured the - * credentials for this repo. - */ connectionConfig?: ConnectionConfig; -} \ No newline at end of file +}; + +export interface JobLifecycleContext { + data: DataOf; + jobId: string; + attemptsMade: number; + maxAttempts: number; + prisma: PrismaClient; + logger: JobLogSink; +} + +export interface ProcessContext + extends JobLifecycleContext { + signal: AbortSignal; + updateProgress(progress: number | object): Promise; + trigger(workload: T, data: DataOf): Promise; +} + +export type Schedule = { every: string } | { pattern: string }; + +/** + * A Workload is a single kind of background work, declared + * as the queue it runs on, the code that processes the job, + * and how much of it may run at once. + * + * Jobs reach a workload's queue in one of two ways: someone calls `trigger`, or - if the + * workload declares a `schedule` - the JobManager enqueues one on that cadence. A sweep is + * just a scheduled workload that carries no payload, and whose `process` scans for work and + * triggers it onto other workloads' queues. + */ +export interface Workload { + queueSpec: QueueSpec; + concurrency: number; + /** + * If set, the JobManager enqueues a job on this cadence rather than waiting for someone to + * `trigger` one. Scheduled jobs carry no payload, so `TData` should be `void`. + */ + schedule?: Schedule; + rateLimit?: { max: number; per: string }; + process(ctx: ProcessContext): Promise; + /** Called before `process` on every attempt. */ + onStarted?(ctx: JobLifecycleContext): Promise; + /** Called after BullMQ marks the job as completed. */ + onCompleted?( + ctx: JobLifecycleContext, + result: TResult, + ): Promise; + /** Called after BullMQ exhausts all attempts and marks the job as failed. */ + onTerminalFailure?( + ctx: JobLifecycleContext, + err: Error, + ): Promise; +} + +export interface JobManager { + register(w: Workload): void; + + start(): Promise; + stop(): Promise; + + trigger( + workload: TName, + data: DataOf, + ): Promise; +} + +export interface QueueCounts { + waiting: number; + active: number; + delayed: number; + completed: number; + failed: number; + paused: number; + prioritized?: number; + "waiting-children"?: number; +} + +export interface JobDetail { + id: string; + name: string; + state: + | "waiting" + | "active" + | "delayed" + | "completed" + | "failed" + | "paused" + | "unknown"; + data: TData; + attemptsMade: number; + maxAttempts: number; + result?: TResult | null; + failedReason?: string | null; + stacktrace?: string[]; + logs: string[]; + enqueuedAt: number; + startedAt: number | null; + finishedAt: number | null; + waitMs?: number | null; + runMs?: number | null; +} diff --git a/packages/backend/src/types/redlock.d.ts b/packages/backend/src/types/redlock.d.ts deleted file mode 100644 index 63c6b5a9b..000000000 --- a/packages/backend/src/types/redlock.d.ts +++ /dev/null @@ -1,95 +0,0 @@ -// Type declarations for redlock -// The redlock package's exports field doesn't include types, so TypeScript can't resolve them. -// This file re-exports the types from the actual .d.ts file. - -declare module 'redlock' { - import { EventEmitter } from "events"; - import { Redis as IORedisClient, Cluster as IORedisCluster } from "ioredis"; - - type Client = IORedisClient | IORedisCluster; - - export type ClientExecutionResult = { - client: Client; - vote: "for"; - value: number; - } | { - client: Client; - vote: "against"; - error: Error; - }; - - export type ExecutionStats = { - readonly membershipSize: number; - readonly quorumSize: number; - readonly votesFor: Set; - readonly votesAgainst: Map; - }; - - export type ExecutionResult = { - attempts: ReadonlyArray>; - }; - - export interface Settings { - readonly driftFactor: number; - readonly retryCount: number; - readonly retryDelay: number; - readonly retryJitter: number; - readonly automaticExtensionThreshold: number; - } - - export class ResourceLockedError extends Error { - readonly message: string; - constructor(message: string); - } - - export class ExecutionError extends Error { - readonly message: string; - readonly attempts: ReadonlyArray>; - constructor(message: string, attempts: ReadonlyArray>); - } - - export class Lock { - readonly redlock: Redlock; - readonly resources: string[]; - readonly value: string; - readonly attempts: ReadonlyArray>; - expiration: number; - constructor(redlock: Redlock, resources: string[], value: string, attempts: ReadonlyArray>, expiration: number); - release(): Promise; - extend(duration: number): Promise; - } - - export type RedlockAbortSignal = AbortSignal & { - error?: Error; - }; - - export default class Redlock extends EventEmitter { - readonly clients: Set; - readonly settings: Settings; - readonly scripts: { - readonly acquireScript: { - value: string; - hash: string; - }; - readonly extendScript: { - value: string; - hash: string; - }; - readonly releaseScript: { - value: string; - hash: string; - }; - }; - constructor(clients: Iterable, settings?: Partial, scripts?: { - readonly acquireScript?: string | ((script: string) => string); - readonly extendScript?: string | ((script: string) => string); - readonly releaseScript?: string | ((script: string) => string); - }); - quit(): Promise; - acquire(resources: string[], duration: number, settings?: Partial): Promise; - release(lock: Lock, settings?: Partial): Promise; - extend(existing: Lock, duration: number, settings?: Partial): Promise; - using(resources: string[], duration: number, settings: Partial, routine?: (signal: RedlockAbortSignal) => Promise): Promise; - using(resources: string[], duration: number, routine: (signal: RedlockAbortSignal) => Promise): Promise; - } -} diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index ba028fb20..7db5130be 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,7 +1,7 @@ import { Logger } from "winston"; import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; -import { env, getTokenFromConfig } from "@sourcebot/shared"; +import { env, getTokenFromConfig, JobLogSink } from "@sourcebot/shared"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; import { GithubAppManager } from "./ee/githubAppManager.js"; @@ -114,7 +114,7 @@ export const fetchWithRetry = async ( // fetch the token here using the connections from the repo. Multiple connections could be referencing this repo, and each // may have their own token. This method will just pick the first connection that has a token (if one exists) and uses that. This // may technically cause syncing to fail if that connection's token just so happens to not have access to the repo it's referencing. -export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: Logger): Promise => { +export const getAuthCredentialsForRepo = async (repo: RepoWithConnections, logger?: JobLogSink): Promise => { if (repo.external_codeHostType === 'github' && env.EXPERIMENT_ASK_GH_GITHUB_TOKEN) { logger?.debug(`Using Ask GitHub PAT for service auth for repo ${repo.displayName} hosted at ${repo.external_codeHostUrl}`); diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index 48e42f300..a15b31ea3 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -31,7 +31,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -51,7 +52,8 @@ const schema = { "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", @@ -215,7 +217,8 @@ const schema = { "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -235,7 +238,8 @@ const schema = { "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index df59a13f4..c18bc9301 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -101,6 +101,7 @@ export interface Settings { */ resyncConnectionIntervalMs?: number; /** + * @deprecated * The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second. */ resyncConnectionPollingIntervalMs?: number; @@ -117,6 +118,7 @@ export interface Settings { */ maxRepoIndexingJobConcurrency?: number; /** + * @deprecated * The number of repo GC jobs to run concurrently. Defaults to 8. */ maxRepoGarbageCollectionJobConcurrency?: number; diff --git a/packages/shared/package.json b/packages/shared/package.json index 2a16111ff..7cb47f380 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -14,11 +14,13 @@ "@google-cloud/secret-manager": "^6.1.1", "@logtail/node": "^0.5.2", "@logtail/winston": "^0.5.2", + "@sentry/node": "^10.40.0", "@sourcebot/db": "workspace:*", "@sourcebot/schemas": "workspace:*", "@t3-oss/env-core": "^0.13.10", "ajv": "^8.17.1", - "ioredis": "^5.4.2", + "bullmq": "^5.81.3", + "ioredis": "^5.11.1", "micromatch": "^4.0.8", "strip-json-comments": "^5.0.1", "triple-beam": "^1.4.1", diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts new file mode 100644 index 000000000..5e5e2e43a --- /dev/null +++ b/packages/shared/src/bullmqClient.ts @@ -0,0 +1,120 @@ +import { Queue } from "bullmq"; +import { randomUUID } from "crypto"; +import { Redis } from "ioredis"; +import { DataOf, QueueName, QueueSpec } from "./queue.js"; +import { readBullMQJobLogs } from "./jobLogger.js"; +import type { GetJobLogsOptions, JobLogs } from "./jobLogger.js"; + +export type WorkloadJobStatus = "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED"; + +export interface WorkloadJob { + id: string; + data: DataOf; + status: WorkloadJobStatus; + errorMessage: string | null; +} + +type WorkloadQueue = Queue< + DataOf, + unknown, + string, + DataOf, + unknown, + string +>; + +const normalizeJobState = (state: string): WorkloadJobStatus | null => { + switch (state) { + case "waiting": + case "waiting-children": + case "delayed": + case "prioritized": + case "paused": + return "PENDING"; + case "active": + return "IN_PROGRESS"; + case "completed": + return "COMPLETED"; + case "failed": + return "FAILED"; + default: + return null; + } +}; + +export class BullMQClient { + private readonly queues = new Map(); + + constructor( + private readonly connection: Redis, + ) {} + + getQueue(spec: QueueSpec): WorkloadQueue { + const queueName = spec.name; + let queue = this.queues.get(queueName); + if (!queue) { + queue = new Queue(queueName, { connection: this.connection }); + this.queues.set(queueName, queue); + } + return queue as WorkloadQueue; + } + + async getJob( + spec: QueueSpec, + jobId: string, + ): Promise | null> { + const job = await this.getQueue(spec).getJob(jobId); + if (!job) { + return null; + } + + const status = normalizeJobState(await job.getState()); + if (!status) { + return null; + } + + return { + id: job.id ?? jobId, + data: job.data as DataOf, + status, + errorMessage: status === "FAILED" ? job.failedReason || null : null, + }; + } + + async getJobLogs( + spec: QueueSpec, + jobId: string, + options: GetJobLogsOptions = {}, + ): Promise { + return readBullMQJobLogs(this.getQueue(spec), jobId, options); + } + + async enqueue( + spec: QueueSpec, + data: DataOf + ): Promise { + const dedupKey = spec.dedupKey?.(data); + const queue = this.getQueue(spec); + + const requestedJobId = randomUUID(); + const job = await queue.add(spec.name, data, { + jobId: requestedJobId, + ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + attempts: spec.jobOptions.attempts, + backoff: { type: spec.jobOptions.backoff.type, delay: spec.jobOptions.backoff.delayMs }, + removeOnComplete: { count: spec.jobOptions.keep.completed }, + removeOnFail: { count: spec.jobOptions.keep.failed }, + keepLogs: spec.jobOptions.keepLogs, + }); + + if (!job.id) { + throw new Error(`BullMQ did not return an id for workload "${spec.name}"`); + } + + return job.id; + } + + async close(): Promise { + await Promise.all([...this.queues.values()].map((queue) => queue.close())); + } +} diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index c299ef1cc..134774a0c 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -25,7 +25,7 @@ export const DEFAULT_CONFIG_SETTINGS: ConfigSettings = { reindexRepoPollingIntervalMs: 1000 * 1, // 1 second maxConnectionSyncJobConcurrency: 8, maxRepoIndexingJobConcurrency: 8, - maxRepoGarbageCollectionJobConcurrency: 8, + maxRepoGarbageCollectionJobConcurrency: 2, repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours enablePublicAccess: false, // deprected, use FORCE_ENABLE_ANONYMOUS_ACCESS instead diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 8e082fe63..d374e9f18 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -397,7 +397,6 @@ const options = { REDIS_TLS_HONOR_CIPHER_ORDER: booleanSchema.optional(), REDIS_TLS_KEY_PASSPHRASE: z.string().optional(), - CONNECTION_MANAGER_UPSERT_TIMEOUT_MS: numberSchema.default(300000), REPO_SYNC_RETRY_BASE_SLEEP_SECONDS: numberSchema.default(60), GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: numberSchema.default(60 * 10), diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 6c1d8d723..d9fe61e1c 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -21,13 +21,11 @@ export type { export * from './lighthouseTypes.js'; export type { RepoMetadata, - RepoIndexingJobMetadata, IdentityProviderType, LicenseStatus, } from "./types.js"; export { repoMetadataSchema, - repoIndexingJobMetadataSchema, } from "./types.js"; export { base64Decode, @@ -94,3 +92,37 @@ export { compareVersions, } from "./versionUtils.js"; export type { Version } from "./versionUtils.js"; +export type { + QueueName, + DataOf, + QueueSpec, +} from "./queue.js" +export { + ACCOUNT_PERMISSION_SYNC_QUEUE, + CONNECTION_QUEUE, + RECONCILIATION_QUEUE, + REPO_INDEX_QUEUE, + REPO_PERMISSION_SYNC_QUEUE, +} from "./queue.js"; +export { + BullMQClient, +} from "./bullmqClient.js"; +export type { + WorkloadJob, + WorkloadJobStatus, +} from "./bullmqClient.js"; +export { + createBullMQJobLogger, + DEFAULT_JOB_LOGS_MAX_ENTRIES, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; +export type { + GetJobLogsOptions, + JobLogEntry, + JobLogFields, + JobLogLevel, + JobLogger, + JobLogs, + JobLogSink, +} from "./jobLogger.js"; diff --git a/packages/shared/src/jobLogger.test.ts b/packages/shared/src/jobLogger.test.ts new file mode 100644 index 000000000..cf727c2ab --- /dev/null +++ b/packages/shared/src/jobLogger.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + applicationLog: vi.fn(), + applicationError: vi.fn(), +})); + +vi.mock("./logger.js", () => ({ + createLogger: vi.fn(() => ({ + log: mocks.applicationLog, + error: mocks.applicationError, + })), +})); + +import { + createBullMQJobLogger, + parseJobLogEntry, + readBullMQJobLogs, +} from "./jobLogger.js"; + +describe("createBullMQJobLogger", () => { + test("writes structured, redacted entries to BullMQ and the application logger", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 1, + log, + }); + + logger.warn("Some repositories were skipped", { + skipped: 2, + accessToken: "do-not-store", + }); + await logger.flush(); + + expect(mocks.applicationLog).toHaveBeenCalledWith( + "warn", + "Some repositories were skipped", + { + skipped: 2, + accessToken: "[REDACTED]", + }, + ); + + const storedEntry = JSON.parse(log.mock.calls[0][0]); + expect(storedEntry).toMatchObject({ + version: 1, + level: "warn", + message: "Some repositories were skipped", + attempt: 2, + fields: { + skipped: 2, + accessToken: "[REDACTED]", + }, + }); + expect(storedEntry.timestamp).toEqual(expect.any(String)); + }); + + test("does not fail the workload when persisting a log entry fails", async () => { + const logger = createBullMQJobLogger({ + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 0, + log: vi.fn().mockRejectedValue(new Error("Redis unavailable")), + }); + + logger.info("Starting"); + await expect(logger.flush()).resolves.toBeUndefined(); + expect(mocks.applicationError).toHaveBeenCalled(); + }); + + test("uses the supplied attempt for post-processing lifecycle logs", async () => { + const log = vi.fn().mockResolvedValue(1); + const logger = createBullMQJobLogger( + { + id: "job-1", + name: "connection", + queueName: "connection", + attemptsMade: 2, + log, + }, + { attempt: 2 }, + ); + + logger.info("Completed"); + await logger.flush(); + + expect(JSON.parse(log.mock.calls[0][0])).toMatchObject({ attempt: 2 }); + }); +}); + +describe("readBullMQJobLogs", () => { + test("parses structured entries and preserves legacy string logs", async () => { + const structuredEntry = JSON.stringify({ + version: 1, + timestamp: "2026-07-28T03:00:00.000Z", + level: "info", + message: "Started", + attempt: 1, + }); + const queue = { + getJobLogs: vi.fn().mockResolvedValue({ + logs: [structuredEntry, "legacy log"], + count: 2, + }), + }; + + const result = await readBullMQJobLogs(queue, "job-1", { + start: 10, + end: 20, + ascending: true, + }); + + expect(queue.getJobLogs).toHaveBeenCalledWith("job-1", 10, 20, true); + expect(result).toEqual({ + logs: [ + parseJobLogEntry(structuredEntry), + { + version: 0, + timestamp: null, + level: "info", + message: "legacy log", + attempt: null, + }, + ], + count: 2, + }); + }); +}); diff --git a/packages/shared/src/jobLogger.ts b/packages/shared/src/jobLogger.ts new file mode 100644 index 000000000..f496cdab6 --- /dev/null +++ b/packages/shared/src/jobLogger.ts @@ -0,0 +1,226 @@ +import type { Job, Queue } from "bullmq"; +import { createLogger } from "./logger.js"; + +export const DEFAULT_JOB_LOGS_MAX_ENTRIES = 500; + +export type JobLogLevel = "debug" | "info" | "warn" | "error"; +export type JobLogFields = Record; + +export interface JobLogEntry { + version: 1 | 0; + timestamp: string | null; + level: JobLogLevel; + message: string; + attempt: number | null; + fields?: JobLogFields; +} + +export interface JobLogSink { + debug(message: string, fields?: unknown): void; + info(message: string, fields?: unknown): void; + warn(message: string, fields?: unknown): void; + error(message: string, fields?: unknown): void; +} + +export interface JobLogger extends JobLogSink { + flush(): Promise; +} + +export interface GetJobLogsOptions { + start?: number; + end?: number; + ascending?: boolean; +} + +export interface JobLogs { + logs: JobLogEntry[]; + count: number; +} + +type BullMQLogJob = Pick< + Job, + "id" | "name" | "queueName" | "attemptsMade" | "log" +>; +type JobLogQueue = Pick; + +const JOB_LOG_LEVELS = new Set(["debug", "info", "warn", "error"]); +const SENSITIVE_FIELD_NAME = + /authorization|cookie|credential|password|private.?key|secret|token/i; +const MAX_FIELD_DEPTH = 6; + +const sanitizeValue = ( + value: unknown, + seen: WeakSet, + depth: number, +): unknown => { + if (depth > MAX_FIELD_DEPTH) { + return "[Max depth reached]"; + } + if ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "bigint") { + return value.toString(); + } + if (typeof value === "undefined") { + return "[undefined]"; + } + if (typeof value === "symbol" || typeof value === "function") { + return String(value); + } + if (value instanceof Date) { + return value.toISOString(); + } + if (value instanceof Error) { + return { + name: value.name, + message: value.message, + stack: value.stack, + }; + } + if (seen.has(value)) { + return "[Circular]"; + } + + seen.add(value); + if (Array.isArray(value)) { + return value.map((item) => sanitizeValue(item, seen, depth + 1)); + } + + return Object.fromEntries( + Object.entries(value).map(([key, nestedValue]) => [ + key, + SENSITIVE_FIELD_NAME.test(key) + ? "[REDACTED]" + : sanitizeValue(nestedValue, seen, depth + 1), + ]), + ); +}; + +const sanitizeFields = (fields: unknown): JobLogFields | undefined => { + if (fields === undefined) { + return undefined; + } + + const sanitized = sanitizeValue(fields, new WeakSet(), 0); + if ( + sanitized !== null && + typeof sanitized === "object" && + !Array.isArray(sanitized) + ) { + return sanitized as JobLogFields; + } + return { value: sanitized }; +}; + +export const parseJobLogEntry = (rawLog: string): JobLogEntry => { + try { + const parsed = JSON.parse(rawLog) as Partial; + if ( + parsed.version === 1 && + typeof parsed.timestamp === "string" && + typeof parsed.level === "string" && + JOB_LOG_LEVELS.has(parsed.level as JobLogLevel) && + typeof parsed.message === "string" && + typeof parsed.attempt === "number" + ) { + return { + version: 1, + timestamp: parsed.timestamp, + level: parsed.level as JobLogLevel, + message: parsed.message, + attempt: parsed.attempt, + ...(parsed.fields ? { fields: parsed.fields } : {}), + }; + } + } catch { + // Older BullMQ logs were stored as plain strings. + } + + return { + version: 0, + timestamp: null, + level: "info", + message: rawLog, + attempt: null, + }; +}; + +export const readBullMQJobLogs = async ( + queue: JobLogQueue, + jobId: string, + options: GetJobLogsOptions = {}, +): Promise => { + const result = await queue.getJobLogs( + jobId, + options.start, + options.end, + options.ascending, + ); + + return { + logs: result.logs.map(parseJobLogEntry), + count: result.count, + }; +}; + +export const createBullMQJobLogger = ( + job: BullMQLogJob, + options: { + label?: string; + attempt?: number; + } = {}, +): JobLogger => { + const label = + options.label ?? `${job.queueName}:job:${job.id ?? "unknown"}`; + const attempt = options.attempt ?? job.attemptsMade + 1; + const applicationLogger = createLogger(label); + const pendingWrites = new Set>(); + + const write = ( + level: JobLogLevel, + message: string, + rawFields?: unknown, + ): void => { + const fields = sanitizeFields(rawFields); + applicationLogger.log(level, message, fields); + + const entry: JobLogEntry = { + version: 1, + timestamp: new Date().toISOString(), + level, + message, + attempt, + ...(fields ? { fields } : {}), + }; + const pendingWrite = job + .log(JSON.stringify(entry)) + .then(() => undefined) + .catch((error: unknown) => { + applicationLogger.error( + `Failed to persist a BullMQ log entry for job ${job.id ?? "unknown"}`, + error, + ); + }); + + pendingWrites.add(pendingWrite); + void pendingWrite.finally(() => { + pendingWrites.delete(pendingWrite); + }); + }; + + return { + debug: (message, fields) => write("debug", message, fields), + info: (message, fields) => write("info", message, fields), + warn: (message, fields) => write("warn", message, fields), + error: (message, fields) => write("error", message, fields), + flush: async () => { + await Promise.all([...pendingWrites]); + }, + }; +}; diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts new file mode 100644 index 000000000..d7b8d0271 --- /dev/null +++ b/packages/shared/src/queue.ts @@ -0,0 +1,89 @@ +import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; + +export type QueueName = keyof QueueRegistry; +export type DataOf = QueueRegistry[TName]; +type EmptyJobData = Record; + +interface QueueRegistry { + reconciliation: EmptyJobData; + "connection-sync": { + connectionId: number; + orgId: number; + }; + "repo-index": { + repoId: number; + type: "INDEX" | "CLEANUP"; + }; + "account-permission-sync": { + accountId: string; + }; + "repo-permission-sync": { + repoId: number; + }; +} + +export const RECONCILIATION_QUEUE: QueueSpec<"reconciliation"> = { + name: "reconciliation", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, +}; + +export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { + name: "connection-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + dedupKey: (data) => `connection:${data.connectionId}`, +}; + +export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { + name: "repo-index", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + dedupKey: (data) => `repo:${data.repoId}`, +}; + +export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = + { + name: "account-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + dedupKey: (data) => `account:${data.accountId}`, + }; + +export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { + name: "repo-permission-sync", + jobOptions: { + attempts: 2, + backoff: { type: "exponential", delayMs: 5000 }, + keep: { completed: 50, failed: 50 }, + keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES, + }, + dedupKey: (data) => `repo:${data.repoId}`, +}; + +export interface QueueSpec { + name: TName; + dedupKey?(data: DataOf): string; + jobOptions: { + attempts: number; + backoff: { type: "fixed" | "exponential"; delayMs: number }; + keep: { completed: number; failed: number }; + keepLogs: number; + }; +} diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 5951bab6b..9f93637c9 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -63,15 +63,6 @@ export const repoMetadataSchema = z.object({ export type RepoMetadata = z.infer; -export const repoIndexingJobMetadataSchema = z.object({ - /** - * A list of revisions that were indexed for the repo. - */ - indexedRevisions: z.array(z.string()).optional(), -}); - -export type RepoIndexingJobMetadata = z.infer; - export type IdentityProviderType = IdentityProviderConfig['provider']; // @see: https://docs.stripe.com/api/subscriptions/object#subscription_object-status diff --git a/packages/web/src/lib/bullmqClient.ts b/packages/web/src/lib/bullmqClient.ts new file mode 100644 index 000000000..a891bf4bd --- /dev/null +++ b/packages/web/src/lib/bullmqClient.ts @@ -0,0 +1,11 @@ +import 'server-only'; + +import { BullMQClient } from '@sourcebot/shared'; +import { getRedisClient } from './redis'; + +let client: BullMQClient | undefined; + +export function getBullMQClient() { + client ??= new BullMQClient(getRedisClient()); + return client; +} diff --git a/schemas/v3/index.json b/schemas/v3/index.json index 874f9f8d5..21499ef0f 100644 --- a/schemas/v3/index.json +++ b/schemas/v3/index.json @@ -30,7 +30,8 @@ "resyncConnectionPollingIntervalMs": { "type": "number", "description": "The polling rate (in milliseconds) at which the db should be checked for connections that need to be re-synced. Defaults to 1 second.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "reindexRepoPollingIntervalMs": { "type": "number", @@ -50,7 +51,8 @@ "maxRepoGarbageCollectionJobConcurrency": { "type": "number", "description": "The number of repo GC jobs to run concurrently. Defaults to 8.", - "minimum": 1 + "minimum": 1, + "deprecated": true }, "repoGarbageCollectionGracePeriodMs": { "type": "number", diff --git a/yarn.lock b/yarn.lock index c5fff52de..e3ee53e54 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1316,6 +1316,38 @@ __metadata: languageName: node linkType: hard +"@bull-board/api@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/api@npm:6.11.2" + dependencies: + redis-info: "npm:^3.1.0" + peerDependencies: + "@bull-board/ui": 6.11.2 + checksum: 10c0/d6a82bdd598d41c4e09dd4e8a001f49e87ed40aa535ca841c5430590f0df206ae49ac8835c2832b00cd6bdb413ef89ea7083a5ffc1b779a307744ea69d83c76d + languageName: node + linkType: hard + +"@bull-board/express@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/express@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" + ejs: "npm:^3.1.10" + express: "npm:^4.21.1 || ^5.0.0" + checksum: 10c0/36cf6fb63f51f095934ba6167be8704f45d881c882d56e11e34f9b60e0dac16bc989553292942797af0f62beac2c10f116a852988dadb1bc07cd06d48e1b8ae3 + languageName: node + linkType: hard + +"@bull-board/ui@npm:6.11.2": + version: 6.11.2 + resolution: "@bull-board/ui@npm:6.11.2" + dependencies: + "@bull-board/api": "npm:6.11.2" + checksum: 10c0/7ddcb49222c09f32f63d9a55f000445cd59dd6c73620f5e30ffcf04cfa8727c91918485a7c3d17c709d2b721b4a8d32d2026c2a7f208953cd3facd602e595f03 + languageName: node + linkType: hard + "@cfworker/json-schema@npm:^4.0.2": version: 4.1.1 resolution: "@cfworker/json-schema@npm:4.1.1" @@ -3039,10 +3071,10 @@ __metadata: languageName: node linkType: hard -"@ioredis/commands@npm:^1.1.1": - version: 1.2.0 - resolution: "@ioredis/commands@npm:1.2.0" - checksum: 10c0/a5d3c29dd84d8a28b7c67a441ac1715cbd7337a7b88649c0f17c345d89aa218578d2b360760017c48149ef8a70f44b051af9ac0921a0622c2b479614c4f65b36 +"@ioredis/commands@npm:1.10.0": + version: 1.10.0 + resolution: "@ioredis/commands@npm:1.10.0" + checksum: 10c0/baf91e62d0e64ef2b5f7ca4413dc2456fe250e87483beac4a1c8ef1fe5ad0d2fcdeb9b89d4556d8ef6c7455c64a964359d729601fdb06b2f4c76c35dd59afa99 languageName: node linkType: hard @@ -3624,44 +3656,44 @@ __metadata: languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3": - version: 3.0.3 - resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3" +"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -8978,6 +9010,9 @@ __metadata: version: 0.0.0-use.local resolution: "@sourcebot/backend@workspace:packages/backend" dependencies: + "@bull-board/api": "npm:6.11.2" + "@bull-board/express": "npm:6.11.2" + "@bull-board/ui": "npm:6.11.2" "@coderabbitai/bitbucket": "npm:^1.1.3" "@gitbeaker/rest": "npm:^40.5.1" "@octokit/app": "npm:^16.1.1" @@ -8994,7 +9029,7 @@ __metadata: "@types/node": "npm:^22.7.5" argparse: "npm:^2.0.1" azure-devops-node-api: "npm:^15.1.1" - bullmq: "npm:^5.34.10" + bullmq: "npm:^5.81.3" chokidar: "npm:^4.0.3" cross-env: "npm:^7.0.3" cross-fetch: "npm:^4.0.0" @@ -9006,14 +9041,13 @@ __metadata: gitea-js: "npm:^1.22.0" glob: "npm:^11.1.0" http-status-codes: "npm:^2.3.0" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" json-schema-to-typescript: "npm:^15.0.4" lowdb: "npm:^7.0.1" micromatch: "npm:^4.0.8" p-limit: "npm:^7.2.0" posthog-node: "npm:^5.24.15" prom-client: "npm:^15.1.3" - redlock: "npm:5.0.0-beta.2" simple-git: "npm:^3.36.0" tsc-watch: "npm:^6.2.0" tsx: "npm:^4.21.0" @@ -9085,14 +9119,16 @@ __metadata: "@google-cloud/secret-manager": "npm:^6.1.1" "@logtail/node": "npm:^0.5.2" "@logtail/winston": "npm:^0.5.2" + "@sentry/node": "npm:^10.40.0" "@sourcebot/db": "workspace:*" "@sourcebot/schemas": "workspace:*" "@t3-oss/env-core": "npm:^0.13.10" "@types/micromatch": "npm:^4.0.9" "@types/node": "npm:^22.7.5" ajv: "npm:^8.17.1" + bullmq: "npm:^5.81.3" cross-env: "npm:^7.0.3" - ioredis: "npm:^5.4.2" + ioredis: "npm:^5.11.1" micromatch: "npm:^4.0.8" strip-json-comments: "npm:^5.0.1" triple-beam: "npm:^1.4.1" @@ -11410,7 +11446,7 @@ __metadata: languageName: node linkType: hard -"async@npm:^3.2.3": +"async@npm:^3.2.3, async@npm:^3.2.6": version: 3.2.6 resolution: "async@npm:3.2.6" checksum: 10c0/36484bb15ceddf07078688d95e27076379cc2f87b10c03b6dd8a83e89475a3c8df5848859dd06a4c95af1e4c16fc973de0171a77f18ea00be899aca2a4f85e70 @@ -11606,19 +11642,19 @@ __metadata: linkType: hard "body-parser@npm:^2.2.1": - version: 2.3.0 - resolution: "body-parser@npm:2.3.0" + version: 2.2.2 + resolution: "body-parser@npm:2.2.2" dependencies: bytes: "npm:^3.1.2" - content-type: "npm:^2.0.0" + content-type: "npm:^1.0.5" debug: "npm:^4.4.3" - http-errors: "npm:^2.0.1" - iconv-lite: "npm:^0.7.2" + http-errors: "npm:^2.0.0" + iconv-lite: "npm:^0.7.0" on-finished: "npm:^2.4.1" - qs: "npm:^6.15.2" - raw-body: "npm:^3.0.2" - type-is: "npm:^2.1.0" - checksum: 10c0/2a8fbbdc471b588338555a3e1a597d1eb0ad0c21cf20fdc3bac5d3f8d9c3a4b19b4163575ab852a43a5dfc0df7770ced5284f979e561f1a24c2f851ce89e695a + qs: "npm:^6.14.1" + raw-body: "npm:^3.0.1" + type-is: "npm:^2.0.1" + checksum: 10c0/95a830a003b38654b75166ca765358aa92ee3d561bf0e41d6ccdde0e1a0c9783cab6b90b20eb635d23172c010b59d3563a137a738e74da4ba714463510d05137 languageName: node linkType: hard @@ -11657,30 +11693,39 @@ __metadata: linkType: hard "brace-expansion@npm:^1.1.13": - version: 1.1.16 - resolution: "brace-expansion@npm:1.1.16" + version: 1.1.14 + resolution: "brace-expansion@npm:1.1.14" dependencies: balanced-match: "npm:^1.0.0" concat-map: "npm:0.0.1" - checksum: 10c0/b2a915bbedbf4e45840d1fb9a4d391bbf26a79475bd134714d3cee34f1f0edb0ce982738028843be5fbaf8039429f71fa487df8c915b6065ced542c83e58fae6 + checksum: 10c0/b6fdac832bc4e36a753658c9ed052c2e1a2be221763b002df25d1efbf7d21724334e726a6cd5eadc72a4b19ec3efb632d629cc003bc9c62f7af7a7915ffa4385 + languageName: node + linkType: hard + +"brace-expansion@npm:^2.0.1": + version: 2.1.4 + resolution: "brace-expansion@npm:2.1.4" + dependencies: + balanced-match: "npm:^1.0.0" + checksum: 10c0/6c0a0e2573eac1dc565b52b1e1bfbeba39bf1830d106ebbc61ff1eaefcf610e9111cd3baa091addd18e33292135c99e019d3184d966389d85dc958fdcdc1449f languageName: node linkType: hard "brace-expansion@npm:^2.0.3": - version: 2.1.2 - resolution: "brace-expansion@npm:2.1.2" + version: 2.1.0 + resolution: "brace-expansion@npm:2.1.0" dependencies: balanced-match: "npm:^1.0.0" - checksum: 10c0/5442ecab84045d21826268bc56c81a6ef4215327ee1f5ee153f67928e93f7a915d130ecec9966623143277fa3cce036ebb50020024bcc4d78853cdd6af9a19f8 + checksum: 10c0/439cedf3e23d7993b37919f1d6fdc653ec21a42437ec3e7460bea9ca8b17edf7a24a633273c31d61aa4335877cf29a443f1871814131c87997a1e6223e1f1502 languageName: node linkType: hard "brace-expansion@npm:^5.0.5": - version: 5.0.7 - resolution: "brace-expansion@npm:5.0.7" + version: 5.0.6 + resolution: "brace-expansion@npm:5.0.6" dependencies: balanced-match: "npm:^4.0.2" - checksum: 10c0/4769109c3c082de178e449a371bcad50d51ab468f644bce2dd9188efe0cf0a080ed102105d7fc8577382cedc45bad7e6443a91bf3d8102264ee8cf927dbaf205 + checksum: 10c0/8c919869b90f61d533b341d3340be5ee4413232ea89b8246cbc2f38eb014f1d8182785c98a006eaf6111d02dc9eeffefdc240d5ac158625b2ed084dccd4bbf9b languageName: node linkType: hard @@ -11714,18 +11759,22 @@ __metadata: languageName: node linkType: hard -"bullmq@npm:^5.34.10": - version: 5.44.3 - resolution: "bullmq@npm:5.44.3" +"bullmq@npm:^5.81.3": + version: 5.81.3 + resolution: "bullmq@npm:5.81.3" dependencies: - cron-parser: "npm:^4.9.0" - ioredis: "npm:^5.4.1" - msgpackr: "npm:^1.11.2" - node-abort-controller: "npm:^3.1.1" - semver: "npm:^7.5.4" - tslib: "npm:^2.0.0" - uuid: "npm:^9.0.0" - checksum: 10c0/2785929ef59645980e3981d6f5e3f7ef25ef0e8488281a7aa48e89c25a89ff5dde0431051f48b2edadd9675c5b780c02577724c33d5d31762009820b8d404cdb + cron-parser: "npm:4.9.0" + ioredis: "npm:5.11.1" + msgpackr: "npm:2.0.5" + node-abort-controller: "npm:3.1.1" + semver: "npm:7.8.5" + tslib: "npm:2.8.1" + peerDependencies: + redis: ">=5.0.0" + peerDependenciesMeta: + redis: + optional: true + checksum: 10c0/e28abf1f37191966d0204e8717040f99807749579ba34e6cd8ac4698f3daa0327be094e52fa52c6aab991f4cd4cdd338b6c950134ec083be166c19f82d96cde1 languageName: node linkType: hard @@ -12029,10 +12078,10 @@ __metadata: languageName: node linkType: hard -"cluster-key-slot@npm:^1.1.0": - version: 1.1.2 - resolution: "cluster-key-slot@npm:1.1.2" - checksum: 10c0/d7d39ca28a8786e9e801eeb8c770e3c3236a566625d7299a47bb71113fb2298ce1039596acb82590e598c52dbc9b1f088c8f587803e697cb58e1867a95ff94d3 +"cluster-key-slot@npm:1.1.1": + version: 1.1.1 + resolution: "cluster-key-slot@npm:1.1.1" + checksum: 10c0/079b1ae86b20e2d53308a877b08de5e830722a45c07810569d0dab4955bed569da33ac9f79998289d014adf02cca7223a0647cb0ee6548a12ab3c4f9beac1377 languageName: node linkType: hard @@ -12483,13 +12532,6 @@ __metadata: languageName: node linkType: hard -"content-type@npm:^2.0.0": - version: 2.0.0 - resolution: "content-type@npm:2.0.0" - checksum: 10c0/491539fff707d7594b0ca4fabcc084bef2a31ffa754ff0a4f80c4377e3963cff0394317f9271c24087596c97fa675bc123d61fa34ffe65b4904e7d3d3098de72 - languageName: node - linkType: hard - "convert-source-map@npm:^2.0.0": version: 2.0.0 resolution: "convert-source-map@npm:2.0.0" @@ -12560,7 +12602,7 @@ __metadata: languageName: node linkType: hard -"cron-parser@npm:^4.9.0": +"cron-parser@npm:4.9.0": version: 4.9.0 resolution: "cron-parser@npm:4.9.0" dependencies: @@ -13178,6 +13220,18 @@ __metadata: languageName: node linkType: hard +"debug@npm:4.4.3, debug@npm:^4.4.3, debug@npm:~4.4.1": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + "debug@npm:^3.2.7": version: 3.2.7 resolution: "debug@npm:3.2.7" @@ -13199,18 +13253,6 @@ __metadata: languageName: node linkType: hard -"debug@npm:^4.4.3, debug@npm:~4.4.1": - version: 4.4.3 - resolution: "debug@npm:4.4.3" - dependencies: - ms: "npm:^2.1.3" - peerDependenciesMeta: - supports-color: - optional: true - checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 - languageName: node - linkType: hard - "debug@npm:~4.3.2": version: 4.3.7 resolution: "debug@npm:4.3.7" @@ -13317,7 +13359,7 @@ __metadata: languageName: node linkType: hard -"denque@npm:^2.1.0": +"denque@npm:2.1.0": version: 2.1.0 resolution: "denque@npm:2.1.0" checksum: 10c0/f9ef81aa0af9c6c614a727cb3bd13c5d7db2af1abf9e6352045b86e85873e629690f6222f4edd49d10e4ccf8f078bbeec0794fafaf61b659c0589d0c511ec363 @@ -13469,14 +13511,14 @@ __metadata: linkType: hard "dompurify@npm:^3.3.2, dompurify@npm:^3.3.3": - version: 3.4.12 - resolution: "dompurify@npm:3.4.12" + version: 3.4.11 + resolution: "dompurify@npm:3.4.11" dependencies: "@types/trusted-types": "npm:^2.0.7" dependenciesMeta: "@types/trusted-types": optional: true - checksum: 10c0/127a13817353d4e20e75a991a9022a04c3af12af7479f68e2b8bee6dbc7330a96edfb8f4ebff96f4f0f24cd43e8dbba46214131f510c3aa87a843bd8b610d0ab + checksum: 10c0/31439481c7e8fc3805d40c376936fd66936620fb1b1a31a2ec097f6165412c37f2d868e082c9ceba62bb37661c1ea132a5db4d5213434317e30df68d4aca9cc9 languageName: node linkType: hard @@ -13590,6 +13632,17 @@ __metadata: languageName: node linkType: hard +"ejs@npm:^3.1.10": + version: 3.1.10 + resolution: "ejs@npm:3.1.10" + dependencies: + jake: "npm:^10.8.5" + bin: + ejs: bin/cli.js + checksum: 10c0/52eade9e68416ed04f7f92c492183340582a36482836b11eab97b159fcdcfdedc62233a1bf0bf5e5e1851c501f2dca0e2e9afd111db2599e4e7f53ee29429ae1 + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.5.73": version: 1.5.123 resolution: "electron-to-chromium@npm:1.5.123" @@ -14596,6 +14649,42 @@ __metadata: languageName: node linkType: hard +"express@npm:^4.21.1 || ^5.0.0, express@npm:^5.2.1": + version: 5.2.1 + resolution: "express@npm:5.2.1" + dependencies: + accepts: "npm:^2.0.0" + body-parser: "npm:^2.2.1" + content-disposition: "npm:^1.0.0" + content-type: "npm:^1.0.5" + cookie: "npm:^0.7.1" + cookie-signature: "npm:^1.2.1" + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + finalhandler: "npm:^2.1.0" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.0" + merge-descriptors: "npm:^2.0.0" + mime-types: "npm:^3.0.0" + on-finished: "npm:^2.4.1" + once: "npm:^1.4.0" + parseurl: "npm:^1.3.3" + proxy-addr: "npm:^2.0.7" + qs: "npm:^6.14.0" + range-parser: "npm:^1.2.1" + router: "npm:^2.2.0" + send: "npm:^1.1.0" + serve-static: "npm:^2.2.0" + statuses: "npm:^2.0.1" + type-is: "npm:^2.0.1" + vary: "npm:^1.1.2" + checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 + languageName: node + linkType: hard + "express@npm:^4.22.2": version: 4.22.2 resolution: "express@npm:4.22.2" @@ -14635,42 +14724,6 @@ __metadata: languageName: node linkType: hard -"express@npm:^5.2.1": - version: 5.2.1 - resolution: "express@npm:5.2.1" - dependencies: - accepts: "npm:^2.0.0" - body-parser: "npm:^2.2.1" - content-disposition: "npm:^1.0.0" - content-type: "npm:^1.0.5" - cookie: "npm:^0.7.1" - cookie-signature: "npm:^1.2.1" - debug: "npm:^4.4.0" - depd: "npm:^2.0.0" - encodeurl: "npm:^2.0.0" - escape-html: "npm:^1.0.3" - etag: "npm:^1.8.1" - finalhandler: "npm:^2.1.0" - fresh: "npm:^2.0.0" - http-errors: "npm:^2.0.0" - merge-descriptors: "npm:^2.0.0" - mime-types: "npm:^3.0.0" - on-finished: "npm:^2.4.1" - once: "npm:^1.4.0" - parseurl: "npm:^1.3.3" - proxy-addr: "npm:^2.0.7" - qs: "npm:^6.14.0" - range-parser: "npm:^1.2.1" - router: "npm:^2.2.0" - send: "npm:^1.1.0" - serve-static: "npm:^2.2.0" - statuses: "npm:^2.0.1" - type-is: "npm:^2.0.1" - vary: "npm:^1.1.2" - checksum: 10c0/45e8c841ad188a41402ddcd1294901e861ee0819f632fb494f2ed344ef9c43315d294d443fb48d594e6586a3b779785120f43321417adaef8567316a55072949 - languageName: node - linkType: hard - "extend@npm:^3.0.0, extend@npm:^3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" @@ -14777,9 +14830,9 @@ __metadata: linkType: hard "fast-uri@npm:^3.1.2": - version: 3.1.4 - resolution: "fast-uri@npm:3.1.4" - checksum: 10c0/f90948821ceb49980f64f89b8216ba498f5957f26035be813526a55b6145d26cbd63ef5618d5205a3292b31edc9c08589749350cd72bd86c7095eb434dceb757 + version: 3.1.2 + resolution: "fast-uri@npm:3.1.2" + checksum: 10c0/5b35641895959f3f7ab7a7b1b5542bded159346f25ec9f256817b206d50b64eda5828e90d605a2e2fc645c90519a7259c2bab2c942ee728c88b88e5be21b090d languageName: node linkType: hard @@ -14891,6 +14944,15 @@ __metadata: languageName: node linkType: hard +"filelist@npm:^1.0.4": + version: 1.0.6 + resolution: "filelist@npm:1.0.6" + dependencies: + minimatch: "npm:^5.0.1" + checksum: 10c0/6ee725bec3e1936d680a45f14439b224d9f7c71658c145addcf551dd82f03d608522eb6b191aa086b392bc3e52ed4ce0ed8d78e24b203e6c5e867560a05d1121 + languageName: node + linkType: hard + "fill-range@npm:^7.1.1": version: 7.1.1 resolution: "fill-range@npm:7.1.1" @@ -15766,9 +15828,9 @@ __metadata: linkType: hard "hono@npm:^4.11.4": - version: 4.12.31 - resolution: "hono@npm:4.12.31" - checksum: 10c0/f80be65cd657cc353b3e478d55424a373d11801b2a1ffc96d94def813c6d852e76bea4437f5df252b15caca98742040d26950bc9c865d8720b89a00a9eac5249 + version: 4.12.25 + resolution: "hono@npm:4.12.25" + checksum: 10c0/9216d647fe2f39b17855b0e74913688b837e3fa9519d367c7beeec399265b36608a820928cc33ab926eee58fe2daf7e33296235b52e56dbfac0fbcd51a5e818e languageName: node linkType: hard @@ -15847,7 +15909,7 @@ __metadata: languageName: node linkType: hard -"http-errors@npm:^2.0.1, http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": +"http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": version: 2.0.1 resolution: "http-errors@npm:2.0.1" dependencies: @@ -15929,7 +15991,7 @@ __metadata: languageName: node linkType: hard -"iconv-lite@npm:^0.7.2, iconv-lite@npm:~0.7.0": +"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.2, iconv-lite@npm:~0.7.0": version: 0.7.2 resolution: "iconv-lite@npm:0.7.2" dependencies: @@ -16092,20 +16154,18 @@ __metadata: languageName: node linkType: hard -"ioredis@npm:^5.4.1, ioredis@npm:^5.4.2": - version: 5.6.0 - resolution: "ioredis@npm:5.6.0" +"ioredis@npm:5.11.1, ioredis@npm:^5.11.1": + version: 5.11.1 + resolution: "ioredis@npm:5.11.1" dependencies: - "@ioredis/commands": "npm:^1.1.1" - cluster-key-slot: "npm:^1.1.0" - debug: "npm:^4.3.4" - denque: "npm:^2.1.0" - lodash.defaults: "npm:^4.2.0" - lodash.isarguments: "npm:^3.1.0" - redis-errors: "npm:^1.2.0" - redis-parser: "npm:^3.0.0" - standard-as-callback: "npm:^2.1.0" - checksum: 10c0/a885e5146640fc448706871290ef424ffa39af561f7ee3cf1590085209a509f85e99082bdaaf3cd32fa66758aea3fc2055d1109648ddca96fac4944bf2092c30 + "@ioredis/commands": "npm:1.10.0" + cluster-key-slot: "npm:1.1.1" + debug: "npm:4.4.3" + denque: "npm:2.1.0" + redis-errors: "npm:1.2.0" + redis-parser: "npm:3.0.0" + standard-as-callback: "npm:2.1.0" + checksum: 10c0/a8b27043cf2c045dfc93f40a32ce24cf9f8b57799a37f4234c4b925c365ccf131629590f94a512f546fda2ba8ed034009c94c4933ecd44c50bc166636d929fd6 languageName: node linkType: hard @@ -16577,6 +16637,19 @@ __metadata: languageName: node linkType: hard +"jake@npm:^10.8.5": + version: 10.9.4 + resolution: "jake@npm:10.9.4" + dependencies: + async: "npm:^3.2.6" + filelist: "npm:^1.0.4" + picocolors: "npm:^1.1.1" + bin: + jake: bin/cli.js + checksum: 10c0/bb52f000340d4a32f1a3893b9abe56ef2b77c25da4dbf2c0c874a8159d082dddda50a5ad10e26060198bd645b928ba8dba3b362710f46a247e335321188c5a9c + languageName: node + linkType: hard + "jiti@npm:2.4.2": version: 2.4.2 resolution: "jiti@npm:2.4.2" @@ -16633,13 +16706,13 @@ __metadata: linkType: hard "js-yaml@npm:^4.1.1": - version: 4.3.0 - resolution: "js-yaml@npm:4.3.0" + version: 4.2.0 + resolution: "js-yaml@npm:4.2.0" dependencies: argparse: "npm:^2.0.1" bin: js-yaml: bin/js-yaml.js - checksum: 10c0/058b30473d6915ca5b4feb11e2f7d4d97242f98d00a798ed48dd90b46b7c640398afe9128c5db22c5300f8c6528fe2a174b9a93f351a70ebc28c6203938d8bff + checksum: 10c0/1916456c118746603b067d74bbcbb0445d9a1d5e474ad4ae775e7b20525bed902e01d9d97dd0c81fcd8d4f596162309d0eb057f4aa38f3e9647f14075e9dea45 languageName: node linkType: hard @@ -17178,11 +17251,11 @@ __metadata: linkType: hard "linkify-it@npm:^5.0.1": - version: 5.0.2 - resolution: "linkify-it@npm:5.0.2" + version: 5.0.1 + resolution: "linkify-it@npm:5.0.1" dependencies: uc.micro: "npm:^2.0.0" - checksum: 10c0/dd70b1735a13d41a2cff0a058ac3771166038f23f6aff004dd53873cf985c64b107902fe0b544a5b3d1ff6e63249cf9c648fb3ae9f285481db48f56887adb0d6 + checksum: 10c0/d06d04f1ed03be131740fc900a5e74ea1f49886b052213599e306d469d5ffe2303db76dd8f771de9f28e2b0b38852de22ec46ae597d245f8b66439b0ceb19b10 languageName: node linkType: hard @@ -17246,20 +17319,6 @@ __metadata: languageName: node linkType: hard -"lodash.defaults@npm:^4.2.0": - version: 4.2.0 - resolution: "lodash.defaults@npm:4.2.0" - checksum: 10c0/d5b77aeb702caa69b17be1358faece33a84497bcca814897383c58b28a2f8dfc381b1d9edbec239f8b425126a3bbe4916223da2a576bb0411c2cefd67df80707 - languageName: node - linkType: hard - -"lodash.isarguments@npm:^3.1.0": - version: 3.1.0 - resolution: "lodash.isarguments@npm:3.1.0" - checksum: 10c0/5e8f95ba10975900a3920fb039a3f89a5a79359a1b5565e4e5b4310ed6ebe64011e31d402e34f577eca983a1fc01ff86c926e3cbe602e1ddfc858fdd353e62d8 - languageName: node - linkType: hard - "lodash.isplainobject@npm:^4.0.6": version: 4.0.6 resolution: "lodash.isplainobject@npm:4.0.6" @@ -17274,7 +17333,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.21": +"lodash@npm:^4.17.11, lodash@npm:^4.17.21": version: 4.18.1 resolution: "lodash@npm:4.18.1" checksum: 10c0/757228fc68805c59789e82185135cf85f05d0b2d3d54631d680ca79ec21944ec8314d4533639a14b8bcfbd97a517e78960933041a5af17ecb693ec6eecb99a27 @@ -18231,6 +18290,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^5.0.1": + version: 5.1.9 + resolution: "minimatch@npm:5.1.9" + dependencies: + brace-expansion: "npm:^2.0.1" + checksum: 10c0/4202718683815a7288b13e470160a4f9560cf392adef4f453927505817e01ef6b3476ecde13cfcaed17e7326dd3b69ad44eb2daeb19a217c5500f9277893f1d6 + languageName: node + linkType: hard + "minimatch@npm:^9.0.4, minimatch@npm:^9.0.5": version: 9.0.9 resolution: "minimatch@npm:9.0.9" @@ -18412,16 +18480,16 @@ __metadata: languageName: node linkType: hard -"msgpackr-extract@npm:^3.0.2": - version: 3.0.3 - resolution: "msgpackr-extract@npm:3.0.3" - dependencies: - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.3" - "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.3" +"msgpackr-extract@npm:^3.0.4": + version: 3.0.4 + resolution: "msgpackr-extract@npm:3.0.4" + dependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.4" node-gyp: "npm:latest" node-gyp-build-optional-packages: "npm:5.2.2" dependenciesMeta: @@ -18439,19 +18507,19 @@ __metadata: optional: true bin: download-msgpackr-prebuilds: bin/download-prebuilds.js - checksum: 10c0/e504fd8bf86a29d7527c83776530ee6dc92dcb0273bb3679fd4a85173efead7f0ee32fb82c8410a13c33ef32828c45f81118ffc0fbed5d6842e72299894623b4 + checksum: 10c0/582a9d17abbf3019e600e948736695056280ce401fd0235ee2474e95f9952208b9f6cce4d0e355b03b7d3c5630e6c3d11fe5fc27fdedb2311cce48de464338d8 languageName: node linkType: hard -"msgpackr@npm:^1.11.2": - version: 1.11.2 - resolution: "msgpackr@npm:1.11.2" +"msgpackr@npm:2.0.5": + version: 2.0.5 + resolution: "msgpackr@npm:2.0.5" dependencies: - msgpackr-extract: "npm:^3.0.2" + msgpackr-extract: "npm:^3.0.4" dependenciesMeta: msgpackr-extract: optional: true - checksum: 10c0/7d2e81ca82c397b2352d470d6bc8f4a967fe4fe14f8fc1fc9906b23009fdfb543999b1ad29c700b8861581e0b6bf903d6f0fefb69a09375cbca6d4d802e6c906 + checksum: 10c0/7ac9820cecd44d24d2ef07994405a277509f004d4fb0a77d04e10e9071d0599e86ad53a9daf673e801518230774bdaca72b1fdcff2b516584ab58bd19362e6ad languageName: node linkType: hard @@ -18489,6 +18557,15 @@ __metadata: languageName: node linkType: hard +"nanoid@npm:^3.3.12": + version: 3.3.12 + resolution: "nanoid@npm:3.3.12" + bin: + nanoid: bin/nanoid.cjs + checksum: 10c0/ba142b7b39e11e80c16dd74b0365d407880c87c1cf7e1480956981ae940ee36060fa5b6f092cd1e315184dd19244c657bd017d03327bd3c62247d691c5e8edfb + languageName: node + linkType: hard + "nanoid@npm:^3.3.16": version: 3.3.16 resolution: "nanoid@npm:3.3.16" @@ -18654,7 +18731,7 @@ __metadata: languageName: node linkType: hard -"node-abort-controller@npm:^3.0.1, node-abort-controller@npm:^3.1.1": +"node-abort-controller@npm:3.1.1": version: 3.1.1 resolution: "node-abort-controller@npm:3.1.1" checksum: 10c0/f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 @@ -19727,7 +19804,7 @@ __metadata: languageName: node linkType: hard -"postcss@npm:^8.4.47, postcss@npm:^8.5.12, postcss@npm:^8.5.15": +"postcss@npm:^8.4.47, postcss@npm:^8.5.12": version: 8.5.22 resolution: "postcss@npm:8.5.22" dependencies: @@ -19738,6 +19815,17 @@ __metadata: languageName: node linkType: hard +"postcss@npm:^8.5.15": + version: 8.5.15 + resolution: "postcss@npm:8.5.15" + dependencies: + nanoid: "npm:^3.3.12" + picocolors: "npm:^1.1.1" + source-map-js: "npm:^1.2.1" + checksum: 10c0/7f2e63ae22fbe43aace1bf652bd99da4e90737c64194d49e51ddc9cd0f9e51ff2861a7d734379b494deffa03a880a5c65eec70bc29ee9ebaa7136dde3eee8f31 + languageName: node + linkType: hard + "postgres-array@npm:~2.0.0": version: 2.0.0 resolution: "postgres-array@npm:2.0.0" @@ -20015,8 +20103,8 @@ __metadata: linkType: hard "protobufjs@npm:^7.3.0, protobufjs@npm:^7.4.0, protobufjs@npm:^7.5.3, protobufjs@npm:^7.5.4": - version: 7.6.5 - resolution: "protobufjs@npm:7.6.5" + version: 7.6.4 + resolution: "protobufjs@npm:7.6.4" dependencies: "@protobufjs/aspromise": "npm:^1.1.2" "@protobufjs/base64": "npm:^1.1.2" @@ -20029,7 +20117,7 @@ __metadata: "@protobufjs/utf8": "npm:^1.1.1" "@types/node": "npm:>=13.7.0" long: "npm:^5.3.2" - checksum: 10c0/863eaca9c6f45bfcfb8787c545f53e9c6696507e328e3981b4643c12d35df7f2826021c10e3fd30bef342c13a8e9d306547bac9c0849ee3bf50f770a12f01dc5 + checksum: 10c0/6403eaa9c5a72cc6450c11f38fefafdde243fd806e7ac606ac8d591bc3fdaec45ae764febf83181a2d9aac51aca624e0f46dec368ceea191f7e85e2d6ccaaf93 languageName: node linkType: hard @@ -20171,7 +20259,7 @@ __metadata: languageName: node linkType: hard -"raw-body@npm:^3.0.2": +"raw-body@npm:^3.0.1": version: 3.0.2 resolution: "raw-body@npm:3.0.2" dependencies: @@ -20640,28 +20728,28 @@ __metadata: languageName: node linkType: hard -"redis-errors@npm:^1.0.0, redis-errors@npm:^1.2.0": +"redis-errors@npm:1.2.0, redis-errors@npm:^1.0.0": version: 1.2.0 resolution: "redis-errors@npm:1.2.0" checksum: 10c0/5b316736e9f532d91a35bff631335137a4f974927bb2fb42bf8c2f18879173a211787db8ac4c3fde8f75ed6233eb0888e55d52510b5620e30d69d7d719c8b8a7 languageName: node linkType: hard -"redis-parser@npm:^3.0.0": - version: 3.0.0 - resolution: "redis-parser@npm:3.0.0" +"redis-info@npm:^3.1.0": + version: 3.1.0 + resolution: "redis-info@npm:3.1.0" dependencies: - redis-errors: "npm:^1.0.0" - checksum: 10c0/ee16ac4c7b2a60b1f42a2cdaee22b005bd4453eb2d0588b8a4939718997ae269da717434da5d570fe0b05030466eeb3f902a58cf2e8e1ca058bf6c9c596f632f + lodash: "npm:^4.17.11" + checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38 languageName: node linkType: hard -"redlock@npm:5.0.0-beta.2": - version: 5.0.0-beta.2 - resolution: "redlock@npm:5.0.0-beta.2" +"redis-parser@npm:3.0.0": + version: 3.0.0 + resolution: "redis-parser@npm:3.0.0" dependencies: - node-abort-controller: "npm:^3.0.1" - checksum: 10c0/6664a1b7807ec0ceb223d8f50bf087ac9a6df6356329c47b4374c3a7e4cbcb24a3045d45735dbc74422fc399657baf533e24c0d6528c141c9dd14701880d4fe4 + redis-errors: "npm:^1.0.0" + checksum: 10c0/ee16ac4c7b2a60b1f42a2cdaee22b005bd4453eb2d0588b8a4939718997ae269da717434da5d570fe0b05030466eeb3f902a58cf2e8e1ca058bf6c9c596f632f languageName: node linkType: hard @@ -21346,6 +21434,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:7.8.5, semver@npm:^7.8.5": + version: 7.8.5 + resolution: "semver@npm:7.8.5" + bin: + semver: bin/semver.js + checksum: 10c0/b1f3127a5be8125a94f37188b361c212466c292c6910adce3ec106cff5dc211ccaedc4739c11bb70fda59d6fc1f040a9bca289f4e093451521a2372e5231fe0c + languageName: node + linkType: hard + "semver@npm:^6.3.1": version: 6.3.1 resolution: "semver@npm:6.3.1" @@ -21355,7 +21452,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.5.4, semver@npm:^7.6.0, semver@npm:^7.6.3": +"semver@npm:^7.3.5, semver@npm:^7.5.3, semver@npm:^7.6.0, semver@npm:^7.6.3": version: 7.7.1 resolution: "semver@npm:7.7.1" bin: @@ -21373,15 +21470,6 @@ __metadata: languageName: node linkType: hard -"semver@npm:^7.8.5": - version: 7.8.5 - resolution: "semver@npm:7.8.5" - bin: - semver: bin/semver.js - checksum: 10c0/b1f3127a5be8125a94f37188b361c212466c292c6910adce3ec106cff5dc211ccaedc4739c11bb70fda59d6fc1f040a9bca289f4e093451521a2372e5231fe0c - languageName: node - linkType: hard - "send@npm:^1.1.0, send@npm:^1.2.0": version: 1.2.0 resolution: "send@npm:1.2.0" @@ -21441,9 +21529,9 @@ __metadata: linkType: hard "seroval@npm:~1.5.0": - version: 1.5.6 - resolution: "seroval@npm:1.5.6" - checksum: 10c0/47e4fb25305bf05fdf300cac6b0d4aaaaf1e12f15c819195afcdf512d88901a3f1f7754ac5a2de740cc0bb04e912c653fbf0129fb3e4c81ce12809e6aa5815e3 + version: 1.5.0 + resolution: "seroval@npm:1.5.0" + checksum: 10c0/aff16b14a7145388555cefd4ebd41759024ee1c2c064080fd8d4fabea4b7c89d103155cd98f5109523b8878e577da73cc6cd8abf98965f2d1f0ba19dc38317ab languageName: node linkType: hard @@ -21663,9 +21751,9 @@ __metadata: linkType: hard "shell-quote@npm:^1.6.1, shell-quote@npm:^1.8.4": - version: 1.10.0 - resolution: "shell-quote@npm:1.10.0" - checksum: 10c0/46ee59bfd972ce6a45500c44ed130dff2d0a7d6fbac9841e59d548518cad8060a06393c9a5dcbc0cede294ad80b2a2cd8c904679e09265f53efc0a0879f30961 + version: 1.8.4 + resolution: "shell-quote@npm:1.8.4" + checksum: 10c0/86c93678bc394cb81f5ddcdc87df9c95d279ef9652775cd1cd1eed361404169a8d8cbaacaeed232ab09919e36ee1e5363863570390d78571f8c22b7f6312fb40 languageName: node linkType: hard @@ -22035,7 +22123,7 @@ __metadata: languageName: node linkType: hard -"standard-as-callback@npm:^2.1.0": +"standard-as-callback@npm:2.1.0": version: 2.1.0 resolution: "standard-as-callback@npm:2.1.0" checksum: 10c0/012677236e3d3fdc5689d29e64ea8a599331c4babe86956bf92fc5e127d53f85411c5536ee0079c52c43beb0026b5ce7aa1d834dd35dd026e82a15d1bcaead1f @@ -22574,15 +22662,15 @@ __metadata: linkType: hard "tar@npm:^7.4.3": - version: 7.5.20 - resolution: "tar@npm:7.5.20" + version: 7.5.16 + resolution: "tar@npm:7.5.16" dependencies: "@isaacs/fs-minipass": "npm:^4.0.0" chownr: "npm:^3.0.0" minipass: "npm:^7.1.2" minizlib: "npm:^3.1.0" yallist: "npm:^5.0.0" - checksum: 10c0/4df4335c6d958b76adf1eaced55889dec3ca1c51f450658a074af80694bb0d9c154a8e93fdf4da617372d1575b121295379993961bbe4cd4b0867c0e5689846a + checksum: 10c0/4f37f3c4bd2ca2755fd736a5df1d573c1a868ec1b1e893346aeafa95ac510f9e2fd1469420bd866cc7904799e5bd4ac62b5d4f03fe27747d6e1e373b44505c5c languageName: node linkType: hard @@ -22966,7 +23054,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": +"tslib@npm:2.8.1, tslib@npm:^2.0.0, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:^2.6.2, tslib@npm:^2.7.0, tslib@npm:^2.8.0, tslib@npm:^2.8.1": version: 2.8.1 resolution: "tslib@npm:2.8.1" checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 @@ -23046,17 +23134,6 @@ __metadata: languageName: node linkType: hard -"type-is@npm:^2.1.0": - version: 2.1.0 - resolution: "type-is@npm:2.1.0" - dependencies: - content-type: "npm:^2.0.0" - media-typer: "npm:^1.1.0" - mime-types: "npm:^3.0.0" - checksum: 10c0/a6018f8f509de48f2c7429305e3a920e73b374fa93127dd0877ae1c2df65a5d33907caac8afb0c37a9b9fc7c49f29e3f55d668963dc845d966930b667c07f50e - languageName: node - linkType: hard - "type-is@npm:~1.6.18": version: 1.6.18 resolution: "type-is@npm:1.6.18"