diff --git a/src/constants/airflow.constant.ts b/src/constants/airflow.constant.ts new file mode 100644 index 00000000..d184a43f --- /dev/null +++ b/src/constants/airflow.constant.ts @@ -0,0 +1,8 @@ +import { PlatformNames } from '@togethercrew.dev/db'; + +export const SUPPORTED_AIRFLOW_PLATFORMS = [PlatformNames.Discourse] as const; +export const AIRFLOW_PLATFORM_INFO = { + [PlatformNames.Discourse]: { + etl: 'discourse_analyzer_etl', + }, +}; diff --git a/src/controllers/platform.controller.ts b/src/controllers/platform.controller.ts index 80220ec2..4b6d384c 100644 --- a/src/controllers/platform.controller.ts +++ b/src/controllers/platform.controller.ts @@ -18,14 +18,14 @@ import config from '../config'; import httpStatus from 'http-status'; import querystring from 'querystring'; import parentLogger from '../config/logger'; -import { PlatformNames } from '@togethercrew.dev/db'; -import { DatabaseManager } from '@togethercrew.dev/db'; +import { PlatformNames, DatabaseManager } from '@togethercrew.dev/db'; const logger = parentLogger.child({ module: 'PlatformController' }); const createPlatform = catchAsync(async function (req: IAuthRequest, res: Response) { const community = req.community; const platform = await platformService.managePlatformConnection(community?.id, req.body); + // await airflowService.triggerDag(platform); res.status(httpStatus.CREATED).send(platform); }); diff --git a/src/docs/module.doc.yml b/src/docs/module.doc.yml index 16f70756..4e54e2be 100644 --- a/src/docs/module.doc.yml +++ b/src/docs/module.doc.yml @@ -18,23 +18,24 @@ paths: type: string enum: - hivemind + - violationDetection community: type: string format: objectId responses: - "201": + '201': description: Module created successfully. content: application/json: schema: - $ref: "#/components/schemas/Module" - "400": + $ref: '#/components/schemas/Module' + '400': description: Bad Request - $ref: "#/components/responses/BadRequest" - "401": + $ref: '#/components/responses/BadRequest' + '401': description: Unauthorized - $ref: "#/components/responses/Unauthorized" - + $ref: '#/components/responses/Unauthorized' + get: tags: - Module @@ -48,12 +49,13 @@ paths: type: string enum: - hivemind + - violationDetection - in: query name: community schema: type: string format: objectId - description: + description: required: true - in: query name: sortBy @@ -75,7 +77,7 @@ paths: default: 1 description: Page number responses: - "200": + '200': description: OK content: application/json: @@ -85,7 +87,7 @@ paths: results: type: array items: - $ref: "#/components/schemas/Module" + $ref: '#/components/schemas/Module' page: type: integer example: 1 @@ -98,9 +100,9 @@ paths: totalResults: type: integer example: 1 - "401": + '401': description: Unauthorized - $ref: "#/components/responses/Unauthorized" + $ref: '#/components/responses/Unauthorized' /api/v1/modules/{moduleId}: get: @@ -117,18 +119,18 @@ paths: schema: type: string responses: - "200": + '200': description: Module details retrieved successfully. content: application/json: schema: - $ref: "#/components/schemas/Module" - "401": + $ref: '#/components/schemas/Module' + '401': description: Unauthorized - $ref: "#/components/responses/Unauthorized" - "404": + $ref: '#/components/responses/Unauthorized' + '404': description: NotFound - $ref: "#/components/responses/NotFound" + $ref: '#/components/responses/NotFound' patch: tags: @@ -165,7 +167,7 @@ paths: name: type: string description: Name of the platform. - enum: ['discord', 'google','github','notion','mediaWiki'] + enum: ['discord', 'google', 'github', 'notion', 'mediaWiki', 'discourse'] metadata: type: object description: Metadata specific to the module and platform. Varies depending on the platform name and module name. @@ -174,7 +176,7 @@ paths: properties: answering: type: object - required: ["selectedChannels"] + required: ['selectedChannels'] properties: selectedChannels: type: array @@ -183,7 +185,7 @@ paths: description: Channels selected for answering queries. learning: type: object - required: ["selectedChannels", "fromDate"] + required: ['selectedChannels', 'fromDate'] properties: selectedChannels: type: array @@ -232,20 +234,37 @@ paths: items: type: string description: Metadata for the hivemind module on MediaWiki. + - type: object + properties: + selectedEmails: + type: array + items: + type: string + fromDate: + type: string + format: date. + toDate: + type: string + format: date. + selectedResources: + type: array + items: + type: integer + description: Metadata for the violation detection module on Discourse. minProperties: 1 responses: - "200": + '200': description: Module updated successfully. content: application/json: schema: - $ref: "#/components/schemas/Module" - "401": + $ref: '#/components/schemas/Module' + '401': description: Unauthorized - $ref: "#/components/responses/Unauthorized" - "404": + $ref: '#/components/responses/Unauthorized' + '404': description: NotFound - $ref: "#/components/responses/NotFound" + $ref: '#/components/responses/NotFound' delete: tags: @@ -261,11 +280,11 @@ paths: schema: type: string responses: - "204": + '204': description: Module deleted successfully. - "401": + '401': description: Unauthorized - $ref: "#/components/responses/Unauthorized" - "404": + $ref: '#/components/responses/Unauthorized' + '404': description: NotFound - $ref: "#/components/responses/NotFound" + $ref: '#/components/responses/NotFound' diff --git a/src/docs/platform.doc.yml b/src/docs/platform.doc.yml index 012e064a..72bc71bf 100644 --- a/src/docs/platform.doc.yml +++ b/src/docs/platform.doc.yml @@ -67,6 +67,7 @@ paths: - github - notion - mediaWiki + - discourse description: Name of the platform to create. Must be one of the supported platforms. community: type: string @@ -170,6 +171,21 @@ paths: type: string default: /w/api.php description: Metadata for MediaWiki. + - type: object + required: [id, period, analyzerStartedAt, resources] + properties: + id: + type: string + example: gov.optimism.io + period: + type: date + analyzerStartedAt: + type: date + resources: + type: array + items: + type: integer + description: Metadata for Discourse. responses: '201': description: Platform created successfully. diff --git a/src/interfaces/Airflow.interface.ts b/src/interfaces/Airflow.interface.ts new file mode 100644 index 00000000..d7f27b9a --- /dev/null +++ b/src/interfaces/Airflow.interface.ts @@ -0,0 +1,9 @@ +import { type ObjectId } from 'mongodb'; +import { type Snowflake } from 'discord.js'; + +export interface IDiscourseDagConfig { + platform_id: ObjectId; + id: Snowflake; + period: Date; + recompute: boolean; +} diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts index ed530111..5f276a5d 100644 --- a/src/interfaces/index.ts +++ b/src/interfaces/index.ts @@ -5,3 +5,4 @@ export * from './Channel.interface'; export * from './Guild.interface'; export * from './Twitter.interface'; export * from './Role.interface'; +export * from './Airflow.interface'; diff --git a/src/services/index.ts b/src/services/index.ts index 6f2c278b..cfb20ce9 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -14,7 +14,6 @@ import googleService from './google'; import githubService from './github'; import notionService from './notion'; import discourseService from './discourse'; - export { userService, authService, diff --git a/src/services/platform.service.ts b/src/services/platform.service.ts index 09351411..054fb61b 100644 --- a/src/services/platform.service.ts +++ b/src/services/platform.service.ts @@ -150,6 +150,8 @@ function getMetadataKey(platformName: string): string { return 'workspace_id'; case PlatformNames.MediaWiki: return 'baseURL'; + case PlatformNames.Discourse: + return 'id'; default: throw new Error('Unsupported platform'); } diff --git a/src/types/airflow.type.ts b/src/types/airflow.type.ts new file mode 100644 index 00000000..b99077df --- /dev/null +++ b/src/types/airflow.type.ts @@ -0,0 +1,3 @@ +import { SUPPORTED_AIRFLOW_PLATFORMS } from '../constants/airflow.constant'; + +export type SupportedAirflowPlatforms = (typeof SUPPORTED_AIRFLOW_PLATFORMS)[number]; diff --git a/src/validations/module.validation.ts b/src/validations/module.validation.ts index 1bc7759d..15034769 100644 --- a/src/validations/module.validation.ts +++ b/src/validations/module.validation.ts @@ -1,6 +1,11 @@ import Joi from 'joi'; import { objectId } from './custom.validation'; -import { PlatformNames, ModuleNames, HivemindPlatformNames } from '@togethercrew.dev/db'; +import { + PlatformNames, + ModuleNames, + HivemindPlatformNames, + ViolationDetectionPlatformNames, +} from '@togethercrew.dev/db'; const createModule = { body: Joi.object().keys({ @@ -110,6 +115,37 @@ const hivemindOptions = () => { }); }; +const violationDetectionMetadata = () => { + return Joi.object().keys({ + selectedEmails: Joi.array().items(Joi.string().email()), + fromDate: Joi.date(), + toDate: Joi.date().valid(null), + selectedResources: Joi.array().items(Joi.number().empty()), + }); +}; + +const violationDetectionOptions = () => { + return Joi.object().keys({ + platforms: Joi.array().items( + Joi.object().keys({ + name: Joi.string() + .required() + .valid(...Object.values(ViolationDetectionPlatformNames)), + platform: Joi.string().custom(objectId).required(), + metadata: Joi.when('name', { + switch: [ + { + is: PlatformNames.Discourse, + then: violationDetectionMetadata(), + }, + ], + otherwise: Joi.any().forbidden(), + }).required(), + }), + ), + }); +}; + const dynamicModuleUpdate = (req: any) => { const moduleName = req.module?.name; const paramsOption = { @@ -127,6 +163,13 @@ const dynamicModuleUpdate = (req: any) => { }), }; break; + case ModuleNames.ViolationDetection: + bodyOption = { + body: Joi.object().required().keys({ + options: violationDetectionOptions(), + }), + }; + break; default: req.allowInput = false; return {}; diff --git a/src/validations/platform.validation.ts b/src/validations/platform.validation.ts index e248321a..44d95d9e 100644 --- a/src/validations/platform.validation.ts +++ b/src/validations/platform.validation.ts @@ -74,6 +74,15 @@ const mediaWikiMetadata = () => { }); }; +const discourseMetadata = () => { + return Joi.object().keys({ + id: Joi.string().required(), + period: Joi.date().required(), + analyzerStartedAt: Joi.date().required(), + resources: Joi.array().items(Joi.number().empty()).required(), + }); +}; + const createPlatform = { body: Joi.object().keys({ name: Joi.string() @@ -106,6 +115,10 @@ const createPlatform = { is: PlatformNames.MediaWiki, then: mediaWikiMetadata(), }, + { + is: PlatformNames.Discourse, + then: discourseMetadata(), + }, ], }).required(), }),