From 71acd572243a2d73c4a5c5fccd5a529b4df7c5e4 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 3 Dec 2020 14:10:51 +0100 Subject: [PATCH 01/61] silence some test logs --- src/__tests__/worker.test.ts | 2 ++ src/celery/worker.ts | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/__tests__/worker.test.ts b/src/__tests__/worker.test.ts index 43f01beb..dd0b391d 100644 --- a/src/__tests__/worker.test.ts +++ b/src/__tests__/worker.test.ts @@ -12,6 +12,8 @@ function advanceOneTick() { let mockServer: PluginsServer beforeEach(() => { + // silence logs + console.info = jest.fn() mockServer = { ...defaultConfig, db: new Pool(), diff --git a/src/celery/worker.ts b/src/celery/worker.ts index 59be3d08..a6a57703 100644 --- a/src/celery/worker.ts +++ b/src/celery/worker.ts @@ -178,11 +178,11 @@ export default class Worker extends Base { public async stop(): Promise { const taskCount = this.activeTasks.size if (taskCount > 0) { - console.log(`In progress: ${taskCount} tasks. Waiting for them to finish.`) + console.info(`In progress: ${taskCount} tasks. Waiting for them to finish.`) await this.whenCurrentJobsFinished() - console.log(`Finished. Shutting down celery worker.`) + console.info(`Finished. Shutting down celery worker.`) } else { - console.log(`No tasks in progress, shutting down celery worker`) + console.info(`No tasks in progress, shutting down celery worker`) } await this.disconnect() From c605b8d68a975a2343cdbda305bbe3ec30fdc0e6 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 3 Dec 2020 14:23:20 +0100 Subject: [PATCH 02/61] admit defeat in the battle with `esModuleInterop` --- src/server.ts | 2 +- src/utils.ts | 2 +- tsconfig.json | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server.ts b/src/server.ts index 40b0da30..6e5cd198 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,6 +1,6 @@ import { Pool } from 'pg' import * as schedule from 'node-schedule' -import * as Redis from 'ioredis' +import Redis from 'ioredis' import { FastifyInstance } from 'fastify' import { PluginsServer, PluginsServerConfig } from './types' import { version } from '../package.json' diff --git a/src/utils.ts b/src/utils.ts index ce1f00d1..487a9cf8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,6 @@ import { Readable } from 'stream' import * as tar from 'tar-stream' -import * as AdmZip from 'adm-zip' +import AdmZip from 'adm-zip' import * as zlib from 'zlib' /** diff --git a/tsconfig.json b/tsconfig.json index 49d55424..fc8ac445 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,6 +7,7 @@ "emitDecoratorMetadata": true, "experimentalDecorators": true, "moduleResolution": "node", + "esModuleInterop": true, "sourceMap": true, "baseUrl": "./src", "outDir": "./dist", From cf0bc5987d9f6b76e59c19cb49dbdf78a578686b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 3 Dec 2020 14:23:51 +0100 Subject: [PATCH 03/61] add basic typescript piscina workers --- package.json | 1 + src/server.ts | 5 +++ src/server/piscina.ts | 15 ++++++++ yarn.lock | 82 +++++++++++++++++++++++++++++++++++++++---- 4 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 src/server/piscina.ts diff --git a/package.json b/package.json index c7330c5e..4667f21c 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "node-fetch": "^2.6.1", "node-schedule": "^1.3.2", "pg": "^8.4.2", + "piscina": "^2.1.0", "posthog-js-lite": "^0.0.5", "posthog-plugins": "^0.2.3", "tar-stream": "^2.1.4", diff --git a/src/server.ts b/src/server.ts index 6e5cd198..56cc6107 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,6 +7,7 @@ import { version } from '../package.json' import { setupPlugins } from './plugins' import { startWorker } from './worker' import { startFastifyInstance, stopFastifyInstance } from './web/server' +import { piscina } from './server/piscina' export const defaultConfig: PluginsServerConfig = { CELERY_DEFAULT_QUEUE: 'celery', @@ -40,6 +41,10 @@ export async function startPluginsServer(config: PluginsServerConfig): Promise { + if (task === 'hello') { + return `hello ${args[0]}!` + } + } +} diff --git a/yarn.lock b/yarn.lock index 96254343..0523795e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@assemblyscript/loader@^0.10.1": + version "0.10.1" + resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06" + integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== + "@babel/cli@^7.0.0": version "7.12.1" resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.12.1.tgz#e08a0b1cb6fcd4b9eb6a606ba5602c5c0fe24a0c" @@ -1275,13 +1280,6 @@ dependencies: "@types/node" "*" -"@types/gunzip-maybe@^1.4.0": - version "1.4.0" - resolved "https://registry.yarnpkg.com/@types/gunzip-maybe/-/gunzip-maybe-1.4.0.tgz#9410fd15ff68eca8907b7b9198e63e2a7c14d511" - integrity sha512-dFP9GrYAR9KhsjTkWJ8q8Gsfql75YIKcg9DuQOj/IrlPzR7W+1zX+cclw1McV82UXAQ+Lpufvgk3e9bC8+HzgA== - dependencies: - "@types/node" "*" - "@types/ioredis@^4.17.7": version "4.17.7" resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.17.7.tgz#5007a56380fd387315efabc6c777c050feabb121" @@ -1428,6 +1426,13 @@ resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.0.tgz#215c231dff736d5ba92410e6d602050cce7e273f" integrity sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ== +"@types/workerpool@^6.0.0": + version "6.0.0" + resolved "https://registry.yarnpkg.com/@types/workerpool/-/workerpool-6.0.0.tgz#068c31191f7df9b3d49ebe348b1eeb601e75e2d3" + integrity sha512-BjbKVHFBWblQ3vZ5yFq29kbM2TsaUaTOwYgVxqnNjMrT6CktVF8AvMxOJZgHGgNbAzP4z8DK+EshyZcYpdvAhQ== + dependencies: + "@types/node" "*" + "@types/yargs-parser@*": version "15.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d" @@ -1860,6 +1865,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= +base64-js@^1.2.0: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + base64-js@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" @@ -2945,6 +2955,11 @@ event-emitter@~0.3.5: d "1" es5-ext "~0.10.14" +eventemitter-asyncresource@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b" + integrity sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ== + exec-sh@^0.3.2: version "0.3.4" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" @@ -3567,6 +3582,20 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +hdr-histogram-js@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/hdr-histogram-js/-/hdr-histogram-js-2.0.1.tgz#ecb1ff2bcb6181c3e93ff4af9472c28c7e97284e" + integrity sha512-uPZxl1dAFnjUFHWLZmt93vUUvtHeaBay9nVNHu38SdOjMSF/4KqJUqa1Seuj08ptU1rEb6AHvB41X8n/zFZ74Q== + dependencies: + "@assemblyscript/loader" "^0.10.1" + base64-js "^1.2.0" + pako "^1.0.3" + +hdr-histogram-percentiles-obj@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-3.0.0.tgz#9409f4de0c2dda78e61de2d9d78b1e9f3cba283c" + integrity sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw== + hosted-git-info@^2.1.4: version "2.8.8" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" @@ -4986,16 +5015,34 @@ next-tick@~1.0.0: resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" integrity sha1-yobR/ogoFpsBICCOPchCS524NCw= +nice-napi@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/nice-napi/-/nice-napi-1.0.2.tgz#dc0ab5a1eac20ce548802fc5686eaa6bc654927b" + integrity sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA== + dependencies: + node-addon-api "^3.0.0" + node-gyp-build "^4.2.2" + nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-addon-api@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.0.2.tgz#04bc7b83fd845ba785bb6eae25bc857e1ef75681" + integrity sha512-+D4s2HCnxPd5PjjI0STKwncjXTUKKqm74MDMz9OPXavjsGmjkvwgLtA5yoxJUdmpj52+2u+RrXgPipahKczMKg== + node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-gyp-build@^4.2.2: + version "4.2.3" + resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739" + integrity sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg== + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -5268,6 +5315,11 @@ packet-reader@1.0.0: resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== +pako@^1.0.3: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -5472,6 +5524,17 @@ pirates@^4.0.1: dependencies: node-modules-regexp "^1.0.0" +piscina@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/piscina/-/piscina-2.1.0.tgz#947d937e0efd0de7a0e8bf10363c2e053c619413" + integrity sha512-3FgX36QyZcU4prKuNKl7/lWlOF3HAv9n7JpCjw09Zbql2KkzXXQ7E5xUS+RV5wV24Rn0r6Lr8jLdtU/cNZHAnA== + dependencies: + eventemitter-asyncresource "^1.0.0" + hdr-histogram-js "^2.0.1" + hdr-histogram-percentiles-obj "^3.0.0" + optionalDependencies: + nice-napi "^1.0.2" + pkg-dir@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b" @@ -7142,6 +7205,11 @@ word-wrap@^1.2.3, word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== +workerpool@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.0.3.tgz#3f80ba4d914fe7bb8d933628c26e5d7ff820c703" + integrity sha512-meU8ZzO+ipcx/njxtKUcbu2K95085q5WYDo8fR6PMW3hCY4driteIsNsEowYV7dzOtvq0HotUKsReJkK8gKXgg== + wrap-ansi@^6.2.0: version "6.2.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" From cc8b50e9d6988f0a296939b1c6e0a02ba0fd4d10 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 3 Dec 2020 14:26:46 +0100 Subject: [PATCH 04/61] es module fix --- src/__tests__/vm.test.ts | 2 +- src/__tests__/worker.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/__tests__/vm.test.ts b/src/__tests__/vm.test.ts index 21503eec..646887e9 100644 --- a/src/__tests__/vm.test.ts +++ b/src/__tests__/vm.test.ts @@ -2,7 +2,7 @@ import { createPluginConfigVM, prepareForRun } from '../vm' import { PluginConfig, PluginsServer, Plugin } from '../types' import { PluginEvent } from 'posthog-plugins' import { defaultConfig } from '../server' -import * as Redis from 'ioredis' +import Redis from 'ioredis' import * as fetch from 'node-fetch' import { Pool } from 'pg' diff --git a/src/__tests__/worker.test.ts b/src/__tests__/worker.test.ts index dd0b391d..bbb86cee 100644 --- a/src/__tests__/worker.test.ts +++ b/src/__tests__/worker.test.ts @@ -1,7 +1,7 @@ import { startWorker } from '../worker' import { defaultConfig } from '../server' import { Pool } from 'pg' -import * as Redis from 'ioredis' +import Redis from 'ioredis' import { PluginsServer } from '../types' import Client from '../celery/client' From 6b316217c8d8dc2375e31f44b2ac13596624756b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 3 Dec 2020 14:46:43 +0100 Subject: [PATCH 05/61] remove rollup and compile directly with typescript --- package.json | 9 +-- rollup.config.js | 45 ----------- src/server.ts | 3 +- yarn.lock | 205 ++--------------------------------------------- 4 files changed, 9 insertions(+), 253 deletions(-) delete mode 100644 rollup.config.js diff --git a/package.json b/package.json index 4667f21c..2f49759a 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "start:dev": "ts-node-dev --exit-child src/index.ts --config \"{\\\"BASE_DIR\\\": \\\"../posthog\\\"}\"", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", - "compile": "rollup -c", + "compile": "tsc -p .", "lint": "eslint .", "lint:fix": "eslint --fix .", "prettier": "prettier --write .", @@ -44,7 +44,6 @@ "@babel/core": "^7.0.0", "@babel/preset-env": "^7.0.0", "@babel/preset-typescript": "^7.8.3", - "@rollup/plugin-json": "^4.1.0", "@types/adm-zip": "^0.4.33", "@types/ioredis": "^4.17.7", "@types/jest": "^26.0.15", @@ -74,12 +73,6 @@ "lint-staged": ">=10.5.1", "prettier": "^2.1.2", "rimraf": "^3.0.0", - "rollup": "^2.22.1", - "rollup-plugin-babel": "^4.4.0", - "rollup-plugin-commonjs": "^10.1.0", - "rollup-plugin-dts": "^1.4.9", - "rollup-plugin-node-resolve": "^5.2.0", - "rollup-plugin-typescript2": "^0.29.0", "supertest": "^6.0.1", "ts-jest": "^26.4.4", "ts-node": "^9.0.0", diff --git a/rollup.config.js b/rollup.config.js deleted file mode 100644 index a4aebe2f..00000000 --- a/rollup.config.js +++ /dev/null @@ -1,45 +0,0 @@ -import babel from 'rollup-plugin-babel' -import commonjs from 'rollup-plugin-commonjs' -import resolve from 'rollup-plugin-node-resolve' -import pkg from './package.json' -import typescript from 'rollup-plugin-typescript2' -import dts from 'rollup-plugin-dts' -import json from '@rollup/plugin-json' - -const extensions = ['.js', '.jsx', '.ts', '.tsx'] - -const external = Object.keys(pkg.dependencies || {}).concat(Object.keys(pkg.peerDependencies || {})) - -export default [ - { - input: './src/index.ts', - output: { - file: pkg.main, - format: 'cjs', - }, - external, - plugins: [ - // Allows node_modules resolution - resolve({ extensions, preferBuiltins: true, mainFields: ['jsnext', 'module', 'main'] }), - // Allow bundling cjs modules. Rollup doesn't understand cjs - commonjs({ - include: 'node_modules/**', - }), - json(), - // Compile TypeScript/JavaScript files - typescript({ - tsconfigOverride: { - compilerOptions: { - module: 'ESNext', - }, - }, - }), - babel({ extensions, include: ['src/**/*'] }), - ], - }, - { - input: './dist/src/index.d.ts', - output: [{ file: 'dist/index.d.ts', format: 'es' }], - plugins: [dts()], - }, -] diff --git a/src/server.ts b/src/server.ts index 56cc6107..78c9326c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,12 +3,13 @@ import * as schedule from 'node-schedule' import Redis from 'ioredis' import { FastifyInstance } from 'fastify' import { PluginsServer, PluginsServerConfig } from './types' -import { version } from '../package.json' import { setupPlugins } from './plugins' import { startWorker } from './worker' import { startFastifyInstance, stopFastifyInstance } from './web/server' import { piscina } from './server/piscina' +const { version } = require('../package.json') + export const defaultConfig: PluginsServerConfig = { CELERY_DEFAULT_QUEUE: 'celery', DATABASE_URL: 'postgres://localhost:5432/posthog', diff --git a/yarn.lock b/yarn.lock index 0523795e..b507c451 100644 --- a/yarn.lock +++ b/yarn.lock @@ -180,7 +180,7 @@ dependencies: "@babel/types" "^7.12.1" -"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.12.1": +"@babel/helper-module-imports@^7.12.1": version "7.12.5" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz#1bfc0229f794988f76ed0a4d4e90860850b54dfb" integrity sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA== @@ -1188,22 +1188,6 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" -"@rollup/plugin-json@^4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@rollup/plugin-json/-/plugin-json-4.1.0.tgz#54e09867ae6963c593844d8bd7a9c718294496f3" - integrity sha512-yfLbTdNS6amI/2OpmbiBoW12vngr5NW2jCJVZSBEz+H5KfUJZ2M7sDjk0U6GOOdCWFVScShte29o9NezJ53TPw== - dependencies: - "@rollup/pluginutils" "^3.0.8" - -"@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b" - integrity sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg== - dependencies: - "@types/estree" "0.0.39" - estree-walker "^1.0.1" - picomatch "^2.2.2" - "@sinonjs/commons@^1.7.0": version "1.8.1" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.1.tgz#e7df00f98a203324f6dc7cc606cad9d4a8ab2217" @@ -1263,16 +1247,6 @@ resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.2.tgz#66ad9331f63fe8a3d3d9d8c6e3906dd10f6446e8" integrity sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog== -"@types/estree@*": - version "0.0.45" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.45.tgz#e9387572998e5ecdac221950dab3e8c3b16af884" - integrity sha512-jnqIUKDUqJbDIUxm0Uj7bnlMnRm1T/eZ9N+AVMqhPgzrba2GhGG5o/jCTwmdPK709nEZsGoMzXEDUjcXHa3W0g== - -"@types/estree@0.0.39": - version "0.0.39" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f" - integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw== - "@types/graceful-fs@^4.1.2": version "4.1.4" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.4.tgz#4ff9f641a7c6d1a3508ff88bc3141b152772e753" @@ -1377,13 +1351,6 @@ resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.1.5.tgz#b6ab3bba29e16b821d84e09ecfaded462b816b00" integrity sha512-UEyp8LwZ4Dg30kVU2Q3amHHyTn1jEdhCIE59ANed76GaT1Vp76DD3ZWSAxgCrw6wJ0TqeoBpqmfUHiUDPs//HQ== -"@types/resolve@0.0.8": - version "0.0.8" - resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-0.0.8.tgz#f26074d238e02659e323ce1a13d041eee280e194" - integrity sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ== - dependencies: - "@types/node" "*" - "@types/stack-utils@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.0.tgz#7036640b4e21cc2f259ae826ce843d277dad8cff" @@ -1426,13 +1393,6 @@ resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.0.tgz#215c231dff736d5ba92410e6d602050cce7e273f" integrity sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ== -"@types/workerpool@^6.0.0": - version "6.0.0" - resolved "https://registry.yarnpkg.com/@types/workerpool/-/workerpool-6.0.0.tgz#068c31191f7df9b3d49ebe348b1eeb601e75e2d3" - integrity sha512-BjbKVHFBWblQ3vZ5yFq29kbM2TsaUaTOwYgVxqnNjMrT6CktVF8AvMxOJZgHGgNbAzP4z8DK+EshyZcYpdvAhQ== - dependencies: - "@types/node" "*" - "@types/yargs-parser@*": version "15.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d" @@ -1999,11 +1959,6 @@ buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" -builtin-modules@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.1.0.tgz#aad97c15131eb76b65b50ef208e7584cd76a7484" - integrity sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw== - cache-base@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2" @@ -2250,11 +2205,6 @@ commander@^6.2.0: resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.0.tgz#b990bfb8ac030aedc6d11bc04d1488ffef56db75" integrity sha512-zP4jEKbe8SHzKJYQmq8Y9gYjtO/POJLgIdKgV7B9qNmABVFVc+ctqSX6iXh4mCpJfRBOabiZ2YKPg8ciDw6C+Q== -commondir@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" - integrity sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs= - compare-versions@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62" @@ -2932,16 +2882,6 @@ estraverse@^5.1.0, estraverse@^5.2.0: resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880" integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ== -estree-walker@^0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362" - integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w== - -estree-walker@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-1.0.1.tgz#31bc5d612c96b704106b477e6dd5d8aa138cb700" - integrity sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg== - esutils@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" @@ -3226,15 +3166,6 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -find-cache-dir@^3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.1.tgz#89b33fad4a4670daa94f855f7fbe31d6d84fe880" - integrity sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ== - dependencies: - commondir "^1.0.1" - make-dir "^3.0.2" - pkg-dir "^4.1.0" - find-my-way@^3.0.5: version "3.0.5" resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-3.0.5.tgz#f71c5ef1b4865401e1b97ba428121a8f55439eec" @@ -3343,15 +3274,6 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== -fs-extra@8.1.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" - integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^4.0.0" - universalify "^0.1.0" - fs-readdir-recursive@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz#e32fc030a2ccee44a6b5371308da54be0b397d27" @@ -3506,7 +3428,7 @@ globby@^11.0.1: merge2 "^1.3.0" slash "^3.0.0" -graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4: +graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.2.4: version "4.2.4" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== @@ -3925,11 +3847,6 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" -is-module@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591" - integrity sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE= - is-nan@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.3.0.tgz#85d1f5482f7051c2019f5673ccebdb06f3b0db03" @@ -3971,13 +3888,6 @@ is-potential-custom-element-name@^1.0.0: resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.0.tgz#0c52e54bcca391bb2c494b21e8626d7336c6e397" integrity sha1-DFLlS8yjkbssSUsh6GJtczbG45c= -is-reference@^1.1.2: - version "1.2.1" - resolved "https://registry.yarnpkg.com/is-reference/-/is-reference-1.2.1.tgz#8b2dac0b371f4bc994fdeaba9eb542d03002d0b7" - integrity sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ== - dependencies: - "@types/estree" "*" - is-regex@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.1.tgz#c6f98aacc546f6cec5468a07b7b153ab564a57b9" @@ -4574,13 +4484,6 @@ json5@^1.0.1: dependencies: minimist "^1.2.0" -jsonfile@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" - integrity sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss= - optionalDependencies: - graceful-fs "^4.1.6" - jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -4798,13 +4701,6 @@ loud-rejection@^1.0.0: currently-unhandled "^0.4.1" signal-exit "^3.0.0" -magic-string@^0.25.2: - version "0.25.7" - resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051" - integrity sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA== - dependencies: - sourcemap-codec "^1.4.4" - make-dir@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5" @@ -4813,7 +4709,7 @@ make-dir@^2.1.0: pify "^4.0.1" semver "^5.6.0" -make-dir@^3.0.0, make-dir@^3.0.2: +make-dir@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== @@ -5473,7 +5369,7 @@ pgpass@1.x: dependencies: split2 "^3.1.1" -picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1, picomatch@^2.2.2: +picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg== @@ -5542,7 +5438,7 @@ pkg-dir@^2.0.0: dependencies: find-up "^2.1.0" -pkg-dir@^4.1.0, pkg-dir@^4.2.0: +pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== @@ -5994,13 +5890,6 @@ resolve-url@^0.2.1: resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= -resolve@1.17.0: - version "1.17.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.17.0.tgz#b25941b54968231cc2d1bb76a79cb7f2c0bf8444" - integrity sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w== - dependencies: - path-parse "^1.0.6" - resolve@^1.0.0: version "1.19.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" @@ -6009,7 +5898,7 @@ resolve@^1.0.0: is-core-module "^2.1.0" path-parse "^1.0.6" -resolve@^1.10.0, resolve@^1.10.1, resolve@^1.11.0, resolve@^1.11.1, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.3.2: +resolve@^1.10.0, resolve@^1.10.1, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.3.2: version "1.18.1" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.18.1.tgz#018fcb2c5b207d2a6424aee361c5a266da8f4130" integrity sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA== @@ -6066,68 +5955,6 @@ rimraf@^3.0.0: dependencies: glob "^7.1.3" -rollup-plugin-babel@^4.4.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-babel/-/rollup-plugin-babel-4.4.0.tgz#d15bd259466a9d1accbdb2fe2fff17c52d030acb" - integrity sha512-Lek/TYp1+7g7I+uMfJnnSJ7YWoD58ajo6Oarhlex7lvUce+RCKRuGRSgztDO3/MF/PuGKmUL5iTHKf208UNszw== - dependencies: - "@babel/helper-module-imports" "^7.0.0" - rollup-pluginutils "^2.8.1" - -rollup-plugin-commonjs@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-commonjs/-/rollup-plugin-commonjs-10.1.0.tgz#417af3b54503878e084d127adf4d1caf8beb86fb" - integrity sha512-jlXbjZSQg8EIeAAvepNwhJj++qJWNJw1Cl0YnOqKtP5Djx+fFGkp3WRh+W0ASCaFG5w1jhmzDxgu3SJuVxPF4Q== - dependencies: - estree-walker "^0.6.1" - is-reference "^1.1.2" - magic-string "^0.25.2" - resolve "^1.11.0" - rollup-pluginutils "^2.8.1" - -rollup-plugin-dts@^1.4.9: - version "1.4.13" - resolved "https://registry.yarnpkg.com/rollup-plugin-dts/-/rollup-plugin-dts-1.4.13.tgz#4f086e84f4fdcc1f49160799ebc66f6b09db292b" - integrity sha512-7mxoQ6PcmCkBE5ZhrjGDL4k42XLy8BkSqpiRi1MipwiGs+7lwi4mQkp2afX+OzzLjJp/TGM8llfe8uayIUhPEw== - optionalDependencies: - "@babel/code-frame" "^7.10.4" - -rollup-plugin-node-resolve@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-5.2.0.tgz#730f93d10ed202473b1fb54a5997a7db8c6d8523" - integrity sha512-jUlyaDXts7TW2CqQ4GaO5VJ4PwwaV8VUGA7+km3n6k6xtOEacf61u0VXwN80phY/evMcaS+9eIeJ9MOyDxt5Zw== - dependencies: - "@types/resolve" "0.0.8" - builtin-modules "^3.1.0" - is-module "^1.0.0" - resolve "^1.11.1" - rollup-pluginutils "^2.8.1" - -rollup-plugin-typescript2@^0.29.0: - version "0.29.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.29.0.tgz#b7ad83f5241dbc5bdf1e98d9c3fca005ffe39e1a" - integrity sha512-YytahBSZCIjn/elFugEGQR5qTsVhxhUwGZIsA9TmrSsC88qroGo65O5HZP/TTArH2dm0vUmYWhKchhwi2wL9bw== - dependencies: - "@rollup/pluginutils" "^3.1.0" - find-cache-dir "^3.3.1" - fs-extra "8.1.0" - resolve "1.17.0" - tslib "2.0.1" - -rollup-pluginutils@^2.8.1: - version "2.8.2" - resolved "https://registry.yarnpkg.com/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz#72f2af0748b592364dbd3389e600e5a9444a351e" - integrity sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ== - dependencies: - estree-walker "^0.6.1" - -rollup@^2.22.1: - version "2.33.1" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.33.1.tgz#802795164164ee63cd47769d8879c33ec8ae0c40" - integrity sha512-uY4O/IoL9oNW8MMcbA5hcOaz6tZTMIh7qJHx/tzIJm+n1wLoY38BLn6fuy7DhR57oNFLMbDQtDeJoFURt5933w== - optionalDependencies: - fsevents "~2.1.2" - rsvp@^4.8.4: version "4.8.5" resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734" @@ -6422,11 +6249,6 @@ source-map@^0.7.3: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383" integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ== -sourcemap-codec@^1.4.4: - version "1.4.8" - resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4" - integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== - spdx-correct@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" @@ -6920,11 +6742,6 @@ tsconfig@^7.0.0: strip-bom "^3.0.0" strip-json-comments "^2.0.0" -tslib@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.1.tgz#410eb0d113e5b6356490eec749603725b021b43e" - integrity sha512-SgIkNheinmEBgx1IUNirK0TUD4X9yjjBRTqqjggWCU3pUEqIk3/Uwl3yRixYKT6WjQuGiwDv4NomL3wqRCj+CQ== - tslib@^1.8.1, tslib@^1.9.0: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" @@ -7038,11 +6855,6 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" -universalify@^0.1.0: - version "0.1.2" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" - integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== - unset-value@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559" @@ -7205,11 +7017,6 @@ word-wrap@^1.2.3, word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -workerpool@^6.0.3: - version "6.0.3" - resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.0.3.tgz#3f80ba4d914fe7bb8d933628c26e5d7ff820c703" - integrity sha512-meU8ZzO+ipcx/njxtKUcbu2K95085q5WYDo8fR6PMW3hCY4driteIsNsEowYV7dzOtvq0HotUKsReJkK8gKXgg== - wrap-ansi@^6.2.0: version "6.2.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" From 9185fdb12cfea034d5fa849828af28a6fc1f35a8 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 3 Dec 2020 15:44:43 +0100 Subject: [PATCH 06/61] fix fetch import error --- src/vm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vm.ts b/src/vm.ts index 55faed62..4ac6ffcc 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -1,5 +1,5 @@ import { VM } from 'vm2' -import * as fetch from 'node-fetch' +import fetch from 'node-fetch' import { createConsole } from './extensions/console' import { PluginsServer, PluginConfig, PluginConfigVMReponse } from './types' import { PluginEvent } from 'posthog-plugins' From 517ddb89a8f9390e14cdd55bcfe204c5b98d7978 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 3 Dec 2020 16:20:52 +0100 Subject: [PATCH 07/61] make piscina work in jest, dist and dev... run plugins through it --- bin/posthog-plugin-server | 2 +- jest.config.js | 2 +- package.json | 6 +- src/server.ts | 70 +++++++++++-------- src/server/piscina.ts | 15 ---- .../__tests__/queue.test.ts} | 17 ++--- src/worker/config.ts | 5 ++ src/worker/piscina.d.ts | 3 + src/worker/piscina.js | 23 ++++++ src/{worker.ts => worker/queue.ts} | 14 ++-- src/worker/worker.ts | 26 +++++++ tsconfig.json | 1 + 12 files changed, 120 insertions(+), 64 deletions(-) delete mode 100644 src/server/piscina.ts rename src/{__tests__/worker.test.ts => worker/__tests__/queue.test.ts} (92%) create mode 100644 src/worker/config.ts create mode 100644 src/worker/piscina.d.ts create mode 100644 src/worker/piscina.js rename src/{worker.ts => worker/queue.ts} (77%) create mode 100644 src/worker/worker.ts diff --git a/bin/posthog-plugin-server b/bin/posthog-plugin-server index 33b59eac..4f6ebc02 100755 --- a/bin/posthog-plugin-server +++ b/bin/posthog-plugin-server @@ -1,3 +1,3 @@ #!/usr/bin/env node -require(__dirname + '/../dist/index.js') +require(__dirname + '/../dist/src/index.js') diff --git a/jest.config.js b/jest.config.js index 392b432c..12004699 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,5 +4,5 @@ module.exports = { clearMocks: true, coverageProvider: 'v8', setupFilesAfterEnv: ['./jest.setup.pg-mock.js', './jest.setup.redis-mock.js', './jest.setup.fetch-mock.js'], - testMatch: ['/src/__tests__/**/*.test.ts'], + testMatch: ['/src/**/__tests__/**/*.test.ts'], } diff --git a/package.json b/package.json index 2f49759a..1cb0ff11 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,12 @@ "name": "posthog-plugin-server", "version": "0.2.0", "description": "PostHog Plugin Server", - "types": "dist/index.d.ts", - "main": "dist/index.js", + "types": "dist/src/index.d.ts", + "main": "dist/src/index.js", "scripts": { "test": "jest", "start": "yarn start:dev", - "start:dist": "node dist/index.js --config \"{\\\"BASE_DIR\\\": \\\"../posthog\\\"}\"", + "start:dist": "node dist/src/index.js --config \"{\\\"BASE_DIR\\\": \\\"../posthog\\\"}\"", "start:dev": "ts-node-dev --exit-child src/index.ts --config \"{\\\"BASE_DIR\\\": \\\"../posthog\\\"}\"", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", diff --git a/src/server.ts b/src/server.ts index 78c9326c..5a71b0a9 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,12 +3,11 @@ import * as schedule from 'node-schedule' import Redis from 'ioredis' import { FastifyInstance } from 'fastify' import { PluginsServer, PluginsServerConfig } from './types' -import { setupPlugins } from './plugins' -import { startWorker } from './worker' +import { startQueue } from './worker/queue' import { startFastifyInstance, stopFastifyInstance } from './web/server' -import { piscina } from './server/piscina' - -const { version } = require('../package.json') +import { version } from '../package.json' +import { makePiscina } from './worker/piscina' +import { PluginEvent } from 'posthog-plugins' export const defaultConfig: PluginsServerConfig = { CELERY_DEFAULT_QUEUE: 'celery', @@ -22,9 +21,7 @@ export const defaultConfig: PluginsServerConfig = { WEB_HOSTNAME: '0.0.0.0', } -export async function startPluginsServer(config: PluginsServerConfig): Promise { - console.info(`⚡ Starting posthog-plugin-server v${version}…`) - +export async function createServer(config: PluginsServerConfig): Promise<[PluginsServer, () => Promise]> { const serverConfig: PluginsServerConfig = { ...defaultConfig, ...config, @@ -42,46 +39,59 @@ export async function startPluginsServer(config: PluginsServerConfig): Promise { + await server.redis.quit() + await server.db.end() + } - await setupPlugins(server) + return [server, closeServer] +} + +export async function startPluginsServer(config: PluginsServerConfig): Promise { + console.info(`⚡ Starting posthog-plugin-server v${version}…`) + + const [server, closeServer] = await createServer(config) + + let piscina = makePiscina(config) + const processEvent = (event: PluginEvent) => piscina.runTask({ task: 'processEvent', args: { event } }) let fastifyInstance: FastifyInstance | null = null - if (!serverConfig.DISABLE_WEB) { - fastifyInstance = await startFastifyInstance(serverConfig.WEB_PORT, serverConfig.WEB_HOSTNAME) + if (!server.DISABLE_WEB) { + fastifyInstance = await startFastifyInstance(server.WEB_PORT, server.WEB_HOSTNAME) } - let stopWorker = startWorker(server) + let stopQueue = startQueue(server, processEvent) - const pubSub = new Redis(serverConfig.REDIS_URL) - pubSub.subscribe(serverConfig.PLUGINS_RELOAD_PUBSUB_CHANNEL) + const pubSub = new Redis(server.REDIS_URL) + pubSub.subscribe(server.PLUGINS_RELOAD_PUBSUB_CHANNEL) pubSub.on('message', async (channel, message) => { - if (channel === serverConfig.PLUGINS_RELOAD_PUBSUB_CHANNEL) { - console.log('Reloading plugins!') - await stopWorker() - await setupPlugins(server) - stopWorker = startWorker(server) + if (channel === server.PLUGINS_RELOAD_PUBSUB_CHANNEL) { + console.log('Reloading plugins! NOT IMPLEMENTED FOR MULTITHREADING!') + await stopQueue() + await piscina.destroy() + + piscina = makePiscina(config) + stopQueue = startQueue(server, processEvent) } }) // every 5 sec set a @posthog-plugin-server/ping redis key const job = schedule.scheduleJob('*/5 * * * * *', function () { - redis.set('@posthog-plugin-server/ping', new Date().toISOString()) - redis.expire('@posthog-plugin-server/ping', 60) + server.redis.set('@posthog-plugin-server/ping', new Date().toISOString()) + server.redis.expire('@posthog-plugin-server/ping', 60) }) console.info(`✅ Started posthog-plugin-server v${version}!`) const closeJobs = async () => { - if (!serverConfig.DISABLE_WEB) { - await stopFastifyInstance(fastifyInstance!) - } - await stopWorker() pubSub.disconnect() schedule.cancelJob(job) - await redis.quit() - await db.end() + + if (!server.DISABLE_WEB) { + await stopFastifyInstance(fastifyInstance!) + } + await stopQueue() + await piscina.destroy() + await closeServer() } for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { diff --git a/src/server/piscina.ts b/src/server/piscina.ts deleted file mode 100644 index e600968f..00000000 --- a/src/server/piscina.ts +++ /dev/null @@ -1,15 +0,0 @@ -import Piscina from 'piscina' -import { isMainThread } from 'worker_threads' - -export const piscina: Piscina = (null as any) as Piscina - -if (isMainThread) { - module.exports = { piscina: new Piscina({ filename: __filename }) } -} else { - console.log('🧵 Starting Piscina Worker Thread') - module.exports = ({ task, args }: { task: string; args: any[] }): any => { - if (task === 'hello') { - return `hello ${args[0]}!` - } - } -} diff --git a/src/__tests__/worker.test.ts b/src/worker/__tests__/queue.test.ts similarity index 92% rename from src/__tests__/worker.test.ts rename to src/worker/__tests__/queue.test.ts index bbb86cee..d5fb942e 100644 --- a/src/__tests__/worker.test.ts +++ b/src/worker/__tests__/queue.test.ts @@ -1,9 +1,10 @@ -import { startWorker } from '../worker' -import { defaultConfig } from '../server' +import { startQueue } from '../queue' +import { defaultConfig } from '../../server' import { Pool } from 'pg' import Redis from 'ioredis' -import { PluginsServer } from '../types' -import Client from '../celery/client' +import { PluginsServer } from '../../types' +import Client from '../../celery/client' +import { runPlugins } from '../../plugins' function advanceOneTick() { return new Promise((resolve) => process.nextTick(resolve)) @@ -63,7 +64,7 @@ test('worker and task passing via redis', async () => { expect(args2).toEqual(args) expect(kwargs2).toEqual({}) - const stopWorker = startWorker(mockServer) + const stopQueue = startQueue(mockServer, (event) => runPlugins(mockServer, event)) await advanceOneTick() await advanceOneTick() @@ -88,7 +89,7 @@ test('worker and task passing via redis', async () => { const queue5 = await mockServer.redis.get(mockServer.CELERY_DEFAULT_QUEUE) await advanceOneTick() - await stopWorker() + await stopQueue() }) test('process multiple tasks', async () => { @@ -121,7 +122,7 @@ test('process multiple tasks', async () => { expect((await mockServer.redis.get(mockServer.PLUGINS_CELERY_QUEUE))!.length).toBe(3) expect(await mockServer.redis.get(mockServer.CELERY_DEFAULT_QUEUE)).toBe(null) - const stopWorker = startWorker(mockServer) + const stopQueue = startQueue(mockServer, (event) => runPlugins(mockServer, event)) await advanceOneTick() expect((await mockServer.redis.get(mockServer.PLUGINS_CELERY_QUEUE))!.length).toBe(2) @@ -146,5 +147,5 @@ test('process multiple tasks', async () => { expect(defaultQueue.map((q) => JSON.parse(q)['headers']['lang']).join('-o-')).toBe('js-o-js-o-js') - await stopWorker() + await stopQueue() }) diff --git a/src/worker/config.ts b/src/worker/config.ts new file mode 100644 index 00000000..1e11a2d7 --- /dev/null +++ b/src/worker/config.ts @@ -0,0 +1,5 @@ +import { PluginsServerConfig } from '../types' + +export function createConfig(serverConfig: PluginsServerConfig, filename: string): Record { + return { filename, workerData: { serverConfig } } +} diff --git a/src/worker/piscina.d.ts b/src/worker/piscina.d.ts new file mode 100644 index 00000000..e818e2e9 --- /dev/null +++ b/src/worker/piscina.d.ts @@ -0,0 +1,3 @@ +import Piscina from 'piscina' +import { PluginsServerConfig } from '../types' +export const makePiscina: (config: PluginsServerConfig) => Piscina diff --git a/src/worker/piscina.js b/src/worker/piscina.js new file mode 100644 index 00000000..839be125 --- /dev/null +++ b/src/worker/piscina.js @@ -0,0 +1,23 @@ +const { isMainThread } = require('worker_threads') + +if (isMainThread) { + const Piscina = require('piscina') + const { createConfig } = require('./config') + module.exports = { makePiscina: (serverConfig) => new Piscina(createConfig(serverConfig, __filename)) } +} else { + console.log('🧵 Starting Piscina Worker Thread') + + if (areWeTestingWithJest()) { + require('ts-node').register() + const { worker } = require('./worker.ts') + module.exports = worker + } else { + const { createWorker } = require('./worker') + const { workerData } = require('piscina') + module.exports = createWorker(workerData.serverConfig) + } +} + +function areWeTestingWithJest() { + return process.env.JEST_WORKER_ID !== undefined +} diff --git a/src/worker.ts b/src/worker/queue.ts similarity index 77% rename from src/worker.ts rename to src/worker/queue.ts index 3d4b74e0..aa1a86a3 100644 --- a/src/worker.ts +++ b/src/worker/queue.ts @@ -1,11 +1,13 @@ -import { runPlugins } from './plugins' -import { PluginsServer } from './types' +import { PluginsServer } from '../types' import { PluginEvent } from 'posthog-plugins' -import Worker from './celery/worker' -import Client from './celery/client' +import Worker from '../celery/worker' +import Client from '../celery/client' -export function startWorker(server: PluginsServer): () => Promise { +export function startQueue( + server: PluginsServer, + processEvent: (event: PluginEvent) => Promise +): () => Promise { const worker = new Worker(server.redis, server.PLUGINS_CELERY_QUEUE) const client = new Client(server.redis, server.CELERY_DEFAULT_QUEUE) @@ -21,7 +23,7 @@ export function startWorker(server: PluginsServer): () => Promise { sent_at?: string ) => { const event = { distinct_id, ip, site_url, team_id, now, sent_at, ...data } as PluginEvent - const processedEvent = await runPlugins(server, event) + const processedEvent = await processEvent(event) if (processedEvent) { const { distinct_id, ip, site_url, team_id, now, sent_at, ...data } = processedEvent client.sendTask('posthog.tasks.process_event.process_event', [], { diff --git a/src/worker/worker.ts b/src/worker/worker.ts new file mode 100644 index 00000000..46cc941c --- /dev/null +++ b/src/worker/worker.ts @@ -0,0 +1,26 @@ +import { runPlugins, setupPlugins } from '../plugins' +import { createServer } from '../server' +import { PluginsServerConfig } from '../types' + +export async function createWorker(config: PluginsServerConfig) { + const [server, closeServer] = await createServer(config) + await setupPlugins(server) + + const closeJobs = async () => { + await closeServer() + } + for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { + process.on(signal, closeJobs) + } + + return async ({ task, args }: { task: string; args: any }): Promise => { + if (task === 'hello') { + return `hello ${args[0]}!` + } + if (task === 'processEvent') { + const processedEvent = await runPlugins(server, args.event) + console.log('processed an event!') + return processedEvent + } + } +} diff --git a/tsconfig.json b/tsconfig.json index fc8ac445..c606b655 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,6 +8,7 @@ "experimentalDecorators": true, "moduleResolution": "node", "esModuleInterop": true, + "allowJs": true, "sourceMap": true, "baseUrl": "./src", "outDir": "./dist", From 27fd502abba769def6e020308db3571f72367a13 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 11:26:56 +0100 Subject: [PATCH 08/61] simplify tests --- src/__tests__/vm.test.ts | 12 +++--------- src/worker/__tests__/queue.test.ts | 13 ++++--------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/__tests__/vm.test.ts b/src/__tests__/vm.test.ts index 646887e9..34d57041 100644 --- a/src/__tests__/vm.test.ts +++ b/src/__tests__/vm.test.ts @@ -1,10 +1,8 @@ import { createPluginConfigVM, prepareForRun } from '../vm' import { PluginConfig, PluginsServer, Plugin } from '../types' import { PluginEvent } from 'posthog-plugins' -import { defaultConfig } from '../server' -import Redis from 'ioredis' +import { createServer, defaultConfig } from '../server' import * as fetch from 'node-fetch' -import { Pool } from 'pg' const defaultEvent = { distinct_id: 'my_id', @@ -41,12 +39,8 @@ const mockConfig: PluginConfig = { vm: null, } -beforeEach(() => { - mockServer = { - ...defaultConfig, - db: new Pool(), - redis: new Redis('redis://mockmockmock/'), - } +beforeEach(async () => { + mockServer = (await createServer(defaultConfig))[0] }) afterEach(async () => { diff --git a/src/worker/__tests__/queue.test.ts b/src/worker/__tests__/queue.test.ts index d5fb942e..8fa021b3 100644 --- a/src/worker/__tests__/queue.test.ts +++ b/src/worker/__tests__/queue.test.ts @@ -1,7 +1,5 @@ import { startQueue } from '../queue' -import { defaultConfig } from '../../server' -import { Pool } from 'pg' -import Redis from 'ioredis' +import { createServer, defaultConfig } from '../../server' import { PluginsServer } from '../../types' import Client from '../../celery/client' import { runPlugins } from '../../plugins' @@ -12,14 +10,11 @@ function advanceOneTick() { let mockServer: PluginsServer -beforeEach(() => { +beforeEach(async () => { // silence logs console.info = jest.fn() - mockServer = { - ...defaultConfig, - db: new Pool(), - redis: new Redis('redis://mockmockmock/'), - } + + mockServer = (await createServer(defaultConfig))[0] }) test('worker and task passing via redis', async () => { From 630cf5840199399b6f5ee34f0b4ba6d21face1fc Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 11:27:13 +0100 Subject: [PATCH 09/61] store plugin status on the `server` --- src/plugins.ts | 44 +++++++++++++++++++++----------------------- src/server.ts | 5 +++++ src/types.ts | 7 +++++++ 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index af22ac0d..6121d991 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -17,11 +17,6 @@ import { getFileFromArchive } from './utils' import { performance } from 'perf_hooks' import { logTime } from './stats' -const plugins = new Map() -const pluginConfigs = new Map() -const pluginConfigsPerTeam = new Map() -let defaultConfigs: PluginConfig[] = [] - export async function setupPlugins(server: PluginsServer): Promise { const { rows: pluginRows }: { rows: Plugin[] } = await server.db.query( "SELECT * FROM posthog_plugin WHERE id in (SELECT plugin_id FROM posthog_pluginconfig WHERE enabled='t' GROUP BY plugin_id)" @@ -29,11 +24,11 @@ export async function setupPlugins(server: PluginsServer): Promise { const foundPlugins = new Map() for (const row of pluginRows) { foundPlugins.set(row.id, true) - plugins.set(row.id, row) + server.plugins.set(row.id, row) } - for (const [id, plugin] of plugins) { + for (const [id, plugin] of server.plugins) { if (!foundPlugins.has(id)) { - plugins.delete(id) + server.plugins.delete(id) } } @@ -58,10 +53,10 @@ export async function setupPlugins(server: PluginsServer): Promise { "SELECT * FROM posthog_pluginconfig WHERE enabled='t'" ) const foundPluginConfigs = new Map() - pluginConfigsPerTeam.clear() - defaultConfigs = [] + server.pluginConfigsPerTeam.clear() + server.defaultConfigs = [] for (const row of pluginConfigRows) { - const plugin = plugins.get(row.plugin_id) + const plugin = server.plugins.get(row.plugin_id) if (!plugin) { continue } @@ -72,32 +67,35 @@ export async function setupPlugins(server: PluginsServer): Promise { attachments: attachmentsPerConfig.get(row.id) || {}, vm: null, } - pluginConfigs.set(row.id, pluginConfig) + server.pluginConfigs.set(row.id, pluginConfig) if (!row.team_id) { - defaultConfigs.push(row) + server.defaultConfigs.push(row) } else { - let teamConfigs = pluginConfigsPerTeam.get(row.team_id) + let teamConfigs = server.pluginConfigsPerTeam.get(row.team_id) if (!teamConfigs) { teamConfigs = [] - pluginConfigsPerTeam.set(row.team_id, teamConfigs) + server.pluginConfigsPerTeam.set(row.team_id, teamConfigs) } teamConfigs.push(pluginConfig) } } - for (const [id, pluginConfig] of pluginConfigs) { + for (const [id, pluginConfig] of server.pluginConfigs) { if (!foundPluginConfigs.has(id)) { - pluginConfigs.delete(id) + server.pluginConfigs.delete(id) } else if (!pluginConfig.vm) { await loadPlugin(server, pluginConfig) } } - if (defaultConfigs.length > 0) { - defaultConfigs.sort((a, b) => a.order - b.order) - for (const teamId of Object.keys(pluginConfigsPerTeam).map((key: string) => parseInt(key))) { - pluginConfigsPerTeam.set(teamId, [...(pluginConfigsPerTeam.get(teamId) || []), ...defaultConfigs]) - pluginConfigsPerTeam.get(teamId)?.sort((a, b) => a.id - b.id) + if (server.defaultConfigs.length > 0) { + server.defaultConfigs.sort((a, b) => a.order - b.order) + for (const teamId of Object.keys(server.pluginConfigsPerTeam).map((key: string) => parseInt(key))) { + server.pluginConfigsPerTeam.set(teamId, [ + ...(server.pluginConfigsPerTeam.get(teamId) || []), + ...server.defaultConfigs, + ]) + server.pluginConfigsPerTeam.get(teamId)?.sort((a, b) => a.id - b.id) } } } @@ -183,7 +181,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr } export async function runPlugins(server: PluginsServer, event: PluginEvent): Promise { - const pluginsToRun = pluginConfigsPerTeam.get(event.team_id) || defaultConfigs + const pluginsToRun = server.pluginConfigsPerTeam.get(event.team_id) || server.defaultConfigs let returnedEvent: PluginEvent | null = event diff --git a/src/server.ts b/src/server.ts index 5a71b0a9..6ce6bb75 100644 --- a/src/server.ts +++ b/src/server.ts @@ -37,6 +37,11 @@ export async function createServer(config: PluginsServerConfig): Promise<[Plugin ...serverConfig, db, redis, + + plugins: new Map(), + pluginConfigs: new Map(), + pluginConfigsPerTeam: new Map(), + defaultConfigs: [], } const closeServer = async () => { diff --git a/src/types.ts b/src/types.ts index d5602f1b..1a0f40ac 100644 --- a/src/types.ts +++ b/src/types.ts @@ -16,8 +16,15 @@ export interface PluginsServerConfig { } export interface PluginsServer extends PluginsServerConfig { + // active connections to postgres and redis db: Pool redis: Redis + + // currently enabled plugin status + plugins: Map + pluginConfigs: Map + pluginConfigsPerTeam: Map + defaultConfigs: PluginConfig[] } export type PluginId = number From 4456126c534e6705702d8842f429a0b2a870cb97 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 11:35:46 +0100 Subject: [PATCH 10/61] config types --- src/worker/config.ts | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/worker/config.ts b/src/worker/config.ts index 1e11a2d7..0d6b282f 100644 --- a/src/worker/config.ts +++ b/src/worker/config.ts @@ -1,5 +1,25 @@ import { PluginsServerConfig } from '../types' +import { TaskQueue } from 'piscina/src/common' -export function createConfig(serverConfig: PluginsServerConfig, filename: string): Record { +// Copy From: node_modules/piscina/src/index.ts -- copied because it's not exported +interface Options { + filename?: string | null + minThreads?: number + maxThreads?: number + idleTimeout?: number + maxQueue?: number | 'auto' + concurrentTasksPerWorker?: number + useAtomics?: boolean + resourceLimits?: any + argv?: string[] + execArgv?: string[] + env?: any + workerData?: any + taskQueue?: TaskQueue + niceIncrement?: number + trackUnmanagedFds?: boolean +} + +export function createConfig(serverConfig: PluginsServerConfig, filename: string): Options { return { filename, workerData: { serverConfig } } } From 2f0a872d34841c79afe39b81c500e9dcf1a8fbe9 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 12:01:22 +0100 Subject: [PATCH 11/61] not all keys needed --- src/server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 6ce6bb75..7c31f1e7 100644 --- a/src/server.ts +++ b/src/server.ts @@ -21,7 +21,9 @@ export const defaultConfig: PluginsServerConfig = { WEB_HOSTNAME: '0.0.0.0', } -export async function createServer(config: PluginsServerConfig): Promise<[PluginsServer, () => Promise]> { +export async function createServer( + config: Partial = {} +): Promise<[PluginsServer, () => Promise]> { const serverConfig: PluginsServerConfig = { ...defaultConfig, ...config, From 6143d11e6202745849d8eac6b177f156ee4203a0 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 14:08:47 +0100 Subject: [PATCH 12/61] add worker concurrency setting --- src/index.ts | 9 +++++++-- src/server.ts | 8 ++++++-- src/types.ts | 1 + src/worker/config.ts | 12 +++++++++++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index c6e11be2..615a5b2b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,14 @@ import * as yargs from 'yargs' import { PluginsServerConfig } from './types' import { startPluginsServer } from './server' +import { makePiscina } from './worker/piscina' type Argv = { config: string disableWeb: boolean webPort: number webHostname: string + concurrency: number } yargs @@ -15,17 +17,20 @@ yargs .option('disable-web', { describe: 'Whether web server should be disabled.', type: 'boolean' }) .option('web-port', { alias: 'p', describe: 'Web server port.', type: 'number' }) .option('web-hostname', { alias: 'h', describe: 'Web server hostname.', type: 'string' }) + .option('concurrency', { describe: 'Concurrenct Worker Threads', type: 'number' }) .help() .command({ command: ['start', '$0'], describe: 'start the server', - handler: ({ config, disableWeb, webPort, webHostname }: Argv) => { + handler: ({ config, disableWeb, webPort, webHostname, concurrency }: Argv) => { const parsedConfig: PluginsServerConfig = { ...(config ? JSON.parse(config) : {}), WEB_HOSTNAME: webHostname, WEB_PORT: webPort, DISABLE_WEB: disableWeb, + WORKER_CONCURRENCY: concurrency, } - startPluginsServer(parsedConfig) + + startPluginsServer(parsedConfig, makePiscina) }, }).argv diff --git a/src/server.ts b/src/server.ts index 7c31f1e7..617e9c5c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,8 +6,8 @@ import { PluginsServer, PluginsServerConfig } from './types' import { startQueue } from './worker/queue' import { startFastifyInstance, stopFastifyInstance } from './web/server' import { version } from '../package.json' -import { makePiscina } from './worker/piscina' import { PluginEvent } from 'posthog-plugins' +import Piscina from 'piscina' export const defaultConfig: PluginsServerConfig = { CELERY_DEFAULT_QUEUE: 'celery', @@ -19,6 +19,7 @@ export const defaultConfig: PluginsServerConfig = { DISABLE_WEB: false, WEB_PORT: 3008, WEB_HOSTNAME: '0.0.0.0', + WORKER_CONCURRENCY: 0, // use all cores } export async function createServer( @@ -54,7 +55,10 @@ export async function createServer( return [server, closeServer] } -export async function startPluginsServer(config: PluginsServerConfig): Promise { +export async function startPluginsServer( + config: PluginsServerConfig, + makePiscina: (config: PluginsServerConfig) => Piscina +): Promise { console.info(`⚡ Starting posthog-plugin-server v${version}…`) const [server, closeServer] = await createServer(config) diff --git a/src/types.ts b/src/types.ts index 1a0f40ac..d1885f7c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -4,6 +4,7 @@ import { PluginEvent, PluginAttachment, PluginConfigSchema } from 'posthog-plugi import { VM, VMScript } from 'vm2' export interface PluginsServerConfig { + WORKER_CONCURRENCY: number CELERY_DEFAULT_QUEUE: string DATABASE_URL: string PLUGINS_CELERY_QUEUE: string diff --git a/src/worker/config.ts b/src/worker/config.ts index 0d6b282f..35d82d37 100644 --- a/src/worker/config.ts +++ b/src/worker/config.ts @@ -21,5 +21,15 @@ interface Options { } export function createConfig(serverConfig: PluginsServerConfig, filename: string): Options { - return { filename, workerData: { serverConfig } } + const config: Options = { + filename, + workerData: { serverConfig }, + } + + if (serverConfig.WORKER_CONCURRENCY && serverConfig.WORKER_CONCURRENCY > 0) { + config.minThreads = serverConfig.WORKER_CONCURRENCY + config.maxThreads = serverConfig.WORKER_CONCURRENCY + } + + return config } From 2ed6feede81c3ad57379ad2e829fa0dd2170280e Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 14:09:10 +0100 Subject: [PATCH 13/61] clean up code --- src/worker/worker.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/worker/worker.ts b/src/worker/worker.ts index 46cc941c..7a2c68b9 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -18,9 +18,7 @@ export async function createWorker(config: PluginsServerConfig) { return `hello ${args[0]}!` } if (task === 'processEvent') { - const processedEvent = await runPlugins(server, args.event) - console.log('processed an event!') - return processedEvent + return await runPlugins(server, args.event) } } } From 92b24454aa1aac8ad75bd5f68529c92f56f021e2 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 14:20:48 +0100 Subject: [PATCH 14/61] fix test worker code --- src/worker/piscina.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/worker/piscina.js b/src/worker/piscina.js index 839be125..34a14e92 100644 --- a/src/worker/piscina.js +++ b/src/worker/piscina.js @@ -9,13 +9,11 @@ if (isMainThread) { if (areWeTestingWithJest()) { require('ts-node').register() - const { worker } = require('./worker.ts') - module.exports = worker - } else { - const { createWorker } = require('./worker') - const { workerData } = require('piscina') - module.exports = createWorker(workerData.serverConfig) } + + const { createWorker } = require('./worker') + const { workerData } = require('piscina') + module.exports = createWorker(workerData.serverConfig) } function areWeTestingWithJest() { From 9b85afa86859c45074cd5a77234ee79349acc4df Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 14:21:13 +0100 Subject: [PATCH 15/61] add a basic benchmark test --- src/worker/__tests__/piscina.test.ts | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 src/worker/__tests__/piscina.test.ts diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts new file mode 100644 index 00000000..384f2d78 --- /dev/null +++ b/src/worker/__tests__/piscina.test.ts @@ -0,0 +1,55 @@ +import { defaultConfig } from '../../server' +import { makePiscina } from '../piscina' +import { PluginEvent } from 'posthog-plugins/src/types' +import { performance } from 'perf_hooks' + +jest.setTimeout(300000) // 300 sec timeout + +test('piscina', async () => { + const piscina = makePiscina({ ...defaultConfig, WORKER_CONCURRENCY: 2 }) + const processEvent = (event: PluginEvent) => piscina.runTask({ task: 'processEvent', args: { event } }) + + function processOneEvent(): Promise { + const defaultEvent = { + distinct_id: 'my_id', + ip: '127.0.0.1', + site_url: 'http://localhost', + team_id: 3, + now: new Date().toISOString(), + event: 'default event', + properties: { key: 'value' }, + } + + return processEvent(defaultEvent) + } + + async function processCountEvents(count: number) { + const startTime = performance.now() + const promises = Array(count) + for (let i = 0; i < count; i++) { + promises[i] = processOneEvent() + } + // this will get heavy for tests > 10k events, should chunk them somehow... + await Promise.all(promises) + + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + + const log = { + eventsPerSecond: 1000 / (ms / count), + events: count, + concurrency: piscina.threads.length, + totalMs: ms, + averageEventMs: ms / count, + } + + console.log(JSON.stringify(log, null, 2)) + } + + console.log('100 event warmup!') + await processCountEvents(100) + + console.log('--- START BENCHMARKING ---') + for (let i = 0; i < 10; i++) { + await processCountEvents(10000) + } +}) From f4ec9a7785c50de2b15e1acc9f302a5e7de4376c Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 14:57:51 +0100 Subject: [PATCH 16/61] refactor test --- src/worker/__tests__/piscina.test.ts | 88 ++++++++++++++++------------ 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index 384f2d78..268b802c 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -5,51 +5,67 @@ import { performance } from 'perf_hooks' jest.setTimeout(300000) // 300 sec timeout -test('piscina', async () => { - const piscina = makePiscina({ ...defaultConfig, WORKER_CONCURRENCY: 2 }) +function processOneEvent(processEvent: (event: PluginEvent) => Promise): Promise { + const defaultEvent = { + distinct_id: 'my_id', + ip: '127.0.0.1', + site_url: 'http://localhost', + team_id: 3, + now: new Date().toISOString(), + event: 'default event', + properties: { key: 'value' }, + } + + return processEvent(defaultEvent) +} + +async function processCountEvents(count: number, piscina: ReturnType) { + const startTime = performance.now() + const promises = Array(count) const processEvent = (event: PluginEvent) => piscina.runTask({ task: 'processEvent', args: { event } }) + for (let i = 0; i < count; i++) { + promises[i] = processOneEvent(processEvent) + } + // this will get heavy for tests > 10k events, should chunk them somehow... + await Promise.all(promises) - function processOneEvent(): Promise { - const defaultEvent = { - distinct_id: 'my_id', - ip: '127.0.0.1', - site_url: 'http://localhost', - team_id: 3, - now: new Date().toISOString(), - event: 'default event', - properties: { key: 'value' }, - } - - return processEvent(defaultEvent) + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + + const log = { + eventsPerSecond: 1000 / (ms / count), + events: count, + concurrency: piscina.threads.length, + totalMs: ms, + averageEventMs: ms / count, } - async function processCountEvents(count: number) { - const startTime = performance.now() - const promises = Array(count) - for (let i = 0; i < count; i++) { - promises[i] = processOneEvent() - } - // this will get heavy for tests > 10k events, should chunk them somehow... - await Promise.all(promises) - - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - - const log = { - eventsPerSecond: 1000 / (ms / count), - events: count, - concurrency: piscina.threads.length, - totalMs: ms, - averageEventMs: ms / count, - } - - console.log(JSON.stringify(log, null, 2)) + console.log(JSON.stringify(log, null, 2)) +} + +test('piscina 2 workers', async () => { + const piscina = makePiscina({ ...defaultConfig, WORKER_CONCURRENCY: 2 }) + + console.log('100 event warmup!') + await processCountEvents(100, piscina) + + console.log('--- START BENCHMARKING ---') + for (let i = 0; i < 10; i++) { + await processCountEvents(10000, piscina) } + await piscina.destroy() +}) + +test('piscina 4 workers', async () => { + const piscina = makePiscina({ ...defaultConfig, WORKER_CONCURRENCY: 4 }) + console.log('100 event warmup!') - await processCountEvents(100) + await processCountEvents(100, piscina) console.log('--- START BENCHMARKING ---') for (let i = 0; i < 10; i++) { - await processCountEvents(10000) + await processCountEvents(10000, piscina) } + + await piscina.destroy() }) From 45c90e963a5571ae020027bea9554162274c2e07 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:06:32 +0100 Subject: [PATCH 17/61] refactor test utils --- src/__tests__/helpers/plugins.ts | 116 +++++++++++++++++++++++++++ src/__tests__/helpers/sqlMock.ts | 12 +++ src/__tests__/plugins.test.ts | 133 +++---------------------------- 3 files changed, 138 insertions(+), 123 deletions(-) create mode 100644 src/__tests__/helpers/plugins.ts create mode 100644 src/__tests__/helpers/sqlMock.ts diff --git a/src/__tests__/helpers/plugins.ts b/src/__tests__/helpers/plugins.ts new file mode 100644 index 00000000..10ef4e6d --- /dev/null +++ b/src/__tests__/helpers/plugins.ts @@ -0,0 +1,116 @@ +import { Plugin, PluginAttachmentDB, PluginConfig } from '../../types' +import fs from 'fs' +import path from 'path' +import os from 'os' +import AdmZip from 'adm-zip' + +export const plugin60: Plugin = { + id: 60, + name: 'test-maxmind-plugin', + description: 'Ingest GeoIP data via MaxMind', + url: 'https://www.npmjs.com/package/posthog-maxmind-plugin', + config_schema: { + localhostIP: { + hint: 'Useful if testing locally', + name: 'IP to use instead of 127.0.0.1', + type: 'string', + order: 2, + default: '', + required: false, + }, + maxmindMmdb: { + hint: 'The "GeoIP2 City" or "GeoLite2 City" database file', + name: 'GeoIP .mddb database', + type: 'attachment', + order: 1, + markdown: + 'Sign up for a [MaxMind.com](https://www.maxmind.com) account, download and extract the database and then upload the `.mmdb` file below', + required: true, + }, + }, + tag: '0.0.2', + archive: createZipBuffer('test-maxmind-plugin', { + indexJs: + 'function processEvent (event) { if (event.properties) { event.properties.processed = true } return event }', + }), + error: undefined, +} + +export const pluginAttachment1: PluginAttachmentDB = { + id: 1, + key: 'maxmindMmdb', + content_type: 'application/octet-stream', + file_name: 'test.txt', + file_size: 4, + contents: Buffer.from('test'), + plugin_config_id: 39, + team_id: 2, +} + +export const pluginConfig39: PluginConfig = { + id: 39, + team_id: 2, + plugin_id: 60, + enabled: true, + order: 0, + config: { localhostIP: '94.224.212.175' }, + error: undefined, +} + +function createZipBuffer(name: string, { indexJs, pluginJson }: { indexJs?: string; pluginJson?: string }): Buffer { + const zip = new AdmZip() + if (indexJs) { + zip.addFile('testplugin/index.js', Buffer.alloc(indexJs.length, indexJs)) + } + if (pluginJson) { + zip.addFile('testplugin/plugin.json', Buffer.alloc(pluginJson.length, pluginJson)) + } else { + zip.addFile( + 'testplugin/plugin.json', + new Buffer( + JSON.stringify({ + name, + description: 'just for testing', + url: 'http://example.com/plugin', + config: {}, + main: 'index.js', + }) + ) + ) + } + return zip.toBuffer() +} + +export const mockPluginWithArchive = (indexJs: string, pluginJson?: string) => ({ + ...plugin60, + archive: createZipBuffer('posthog-maxmind-plugin', { indexJs, pluginJson }), +}) + +export const mockJestWithIndex = (indexJs: string) => ({ + getPluginRows: [mockPluginWithArchive(indexJs)], + getPluginConfigRows: [pluginConfig39], + getPluginAttachmentRows: [pluginAttachment1], +}) + +export function mockPluginTempFolder(indexJs: string, pluginJson?: string): [Plugin, () => void] { + const folder = fs.mkdtempSync(path.join(os.tmpdir(), 'foo-')) + + fs.writeFileSync(path.join(folder, 'index.js'), indexJs) + fs.writeFileSync( + path.join(folder, 'plugin.json'), + pluginJson || + JSON.stringify({ + name: 'posthog-maxmind-plugin', + description: 'just for testing', + url: 'http://example.com/plugin', + config: {}, + main: 'index.js', + }) + ) + return [ + { ...plugin60, url: `file:${folder}`, archive: null }, + () => { + fs.rmdirSync(folder, { recursive: true }) + }, + ] +} diff --git a/src/__tests__/helpers/sqlMock.ts b/src/__tests__/helpers/sqlMock.ts new file mode 100644 index 00000000..b6e820cb --- /dev/null +++ b/src/__tests__/helpers/sqlMock.ts @@ -0,0 +1,12 @@ +import * as s from '../../sql' + +// mock functions that get data from postgres and give them the right types +type UnPromisify = F extends (...args: infer A) => Promise ? (...args: A) => T : never +export const getPluginRows = (s.getPluginRows as unknown) as jest.MockedFunction> +export const getPluginAttachmentRows = (s.getPluginAttachmentRows as unknown) as jest.MockedFunction< + UnPromisify +> +export const getPluginConfigRows = (s.getPluginConfigRows as unknown) as jest.MockedFunction< + UnPromisify +> +export const setError = (s.setError as unknown) as jest.MockedFunction> diff --git a/src/__tests__/plugins.test.ts b/src/__tests__/plugins.test.ts index 7663abbc..0c0ed7f3 100644 --- a/src/__tests__/plugins.test.ts +++ b/src/__tests__/plugins.test.ts @@ -1,129 +1,16 @@ import { runPlugins, setupPlugins } from '../plugins' import { createServer } from '../server' -import { Plugin, PluginAttachmentDB, PluginConfig, PluginsServer } from '../types' -import * as s from '../sql' -import AdmZip from 'adm-zip' +import { PluginsServer } from '../types' import { PluginEvent } from 'posthog-plugins/src/types' -import * as fs from 'fs' -import * as os from 'os' -import * as path from 'path' - -// mock functions that get data from postgres and give them the right types +import { + mockPluginTempFolder, + mockPluginWithArchive, + plugin60, + pluginAttachment1, + pluginConfig39, +} from './helpers/plugins' +import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } from './helpers/sqlMock' jest.mock('../sql') -type UnPromisify = F extends (...args: infer A) => Promise ? (...args: A) => T : never -const getPluginRows = (s.getPluginRows as unknown) as jest.MockedFunction> -const getPluginAttachmentRows = (s.getPluginAttachmentRows as unknown) as jest.MockedFunction< - UnPromisify -> -const getPluginConfigRows = (s.getPluginConfigRows as unknown) as jest.MockedFunction< - UnPromisify -> -const setError = (s.setError as unknown) as jest.MockedFunction> - -function createZipBuffer(name: string, { indexJs, pluginJson }: { indexJs?: string; pluginJson?: string }): Buffer { - const zip = new AdmZip() - if (indexJs) { - zip.addFile('testplugin/index.js', Buffer.alloc(indexJs.length, indexJs)) - } - if (pluginJson) { - zip.addFile('testplugin/plugin.json', Buffer.alloc(pluginJson.length, pluginJson)) - } else { - zip.addFile( - 'testplugin/plugin.json', - new Buffer( - JSON.stringify({ - name, - description: 'just for testing', - url: 'http://example.com/plugin', - config: {}, - main: 'index.js', - }) - ) - ) - } - return zip.toBuffer() -} - -const mockPluginWithArchive = (indexJs: string, pluginJson?: string) => ({ - ...plugin60, - archive: createZipBuffer('posthog-maxmind-plugin', { indexJs, pluginJson }), -}) - -function mockPluginTempFolder(indexJs: string, pluginJson?: string): [Plugin, () => void] { - const folder = fs.mkdtempSync(path.join(os.tmpdir(), 'foo-')) - - fs.writeFileSync(path.join(folder, 'index.js'), indexJs) - fs.writeFileSync( - path.join(folder, 'plugin.json'), - pluginJson || - JSON.stringify({ - name: 'posthog-maxmind-plugin', - description: 'just for testing', - url: 'http://example.com/plugin', - config: {}, - main: 'index.js', - }) - ) - return [ - { ...plugin60, url: `file:${folder}`, archive: null }, - () => { - fs.rmdirSync(folder, { recursive: true }) - }, - ] -} - -const plugin60: Plugin = { - id: 60, - name: 'posthog-maxmind-plugin', - description: 'Ingest GeoIP data via MaxMind', - url: 'https://www.npmjs.com/package/posthog-maxmind-plugin', - config_schema: { - localhostIP: { - hint: 'Useful if testing locally', - name: 'IP to use instead of 127.0.0.1', - type: 'string', - order: 2, - default: '', - required: false, - }, - maxmindMmdb: { - hint: 'The "GeoIP2 City" or "GeoLite2 City" database file', - name: 'GeoIP .mddb database', - type: 'attachment', - order: 1, - markdown: - 'Sign up for a [MaxMind.com](https://www.maxmind.com) account, download and extract the database and then upload the `.mmdb` file below', - required: true, - }, - }, - tag: '0.0.2', - archive: createZipBuffer('posthog-maxmind-plugin', { - indexJs: - 'function processEvent (event) { if (event.properties) { event.properties.processed = true } return event }', - }), - error: undefined, -} - -const pluginAttachment1: PluginAttachmentDB = { - id: 1, - key: 'maxmindMmdb', - content_type: 'application/octet-stream', - file_name: 'test.txt', - file_size: 4, - contents: Buffer.from('test'), - plugin_config_id: 39, - team_id: 2, -} - -const pluginConfig39: PluginConfig = { - id: 39, - team_id: 2, - plugin_id: 60, - enabled: true, - order: 0, - config: { localhostIP: '94.224.212.175' }, - error: undefined, -} let mockServer: PluginsServer beforeEach(async () => { @@ -299,7 +186,7 @@ test('archive plugin with broken plugin.json does not do much', async () => { expect(setError).toHaveBeenCalled() expect(setError.mock.calls[0][0]).toEqual(mockServer) - expect(setError.mock.calls[0][1]!.message).toEqual('Can not load plugin.json for plugin "posthog-maxmind-plugin"') + expect(setError.mock.calls[0][1]!.message).toEqual('Can not load plugin.json for plugin "test-maxmind-plugin"') expect(setError.mock.calls[0][1]!.time).toBeDefined() expect(pluginConfigs.get(39)!.vm).toEqual(null) }) From 4eec9e6660ad37d7a5ba7318f53714a444c830f7 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:06:55 +0100 Subject: [PATCH 18/61] support more serialized binary formats --- src/plugins.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index 50dc7de2..94585670 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -141,7 +141,8 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr } } else if (plugin.archive) { let config: PluginJsonConfig = {} - const json = await getFileFromArchive(plugin.archive, 'plugin.json') + const archive = Buffer.from(plugin.archive) + const json = await getFileFromArchive(archive, 'plugin.json') if (json) { try { config = JSON.parse(json) @@ -151,8 +152,8 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr } } - const indexJs = await getFileFromArchive(plugin.archive, config['main'] || 'index.js') - const libJs = await getFileFromArchive(plugin.archive, config['lib'] || 'lib.js') + const indexJs = await getFileFromArchive(archive, config['main'] || 'index.js') + const libJs = await getFileFromArchive(archive, config['lib'] || 'lib.js') if (libJs) { console.warn(`⚠️ Using "lib.js" is deprecated! Used by: ${plugin.name} (${plugin.url})`) } From 90b091379ce4683cba2df5631ea464e0333f54ef Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:08:27 +0100 Subject: [PATCH 19/61] mock jest behind enemy lines --- src/sql.ts | 15 +++++++++++++++ src/types.ts | 2 ++ 2 files changed, 17 insertions(+) diff --git a/src/sql.ts b/src/sql.ts index 028f980b..55911951 100644 --- a/src/sql.ts +++ b/src/sql.ts @@ -1,6 +1,15 @@ import { Plugin, PluginAttachmentDB, PluginConfig, PluginError, PluginsServer } from './types' +// This nice "mocking" system is used since we want to mock data in forked worker threads +// Jest mocks don't penetrate that far. Improvements welcome. +function areWeTestingWithJest() { + return process.env.JEST_WORKER_ID !== undefined +} + export async function getPluginRows(server: PluginsServer): Promise { + if (areWeTestingWithJest() && server.__jestMock?.getPluginRows) { + return server.__jestMock?.getPluginRows + } const { rows: pluginRows }: { rows: Plugin[] } = await server.db.query( "SELECT * FROM posthog_plugin WHERE id in (SELECT plugin_id FROM posthog_pluginconfig WHERE enabled='t' GROUP BY plugin_id)" ) @@ -8,6 +17,9 @@ export async function getPluginRows(server: PluginsServer): Promise { } export async function getPluginAttachmentRows(server: PluginsServer): Promise { + if (areWeTestingWithJest() && server.__jestMock?.getPluginAttachmentRows) { + return server.__jestMock?.getPluginAttachmentRows + } const { rows }: { rows: PluginAttachmentDB[] } = await server.db.query( "SELECT * FROM posthog_pluginattachment WHERE plugin_config_id in (SELECT id FROM posthog_pluginconfig WHERE enabled='t')" ) @@ -15,6 +27,9 @@ export async function getPluginAttachmentRows(server: PluginsServer): Promise { + if (areWeTestingWithJest() && server.__jestMock?.getPluginConfigRows) { + return server.__jestMock?.getPluginConfigRows + } const { rows }: { rows: PluginConfig[] } = await server.db.query( "SELECT * FROM posthog_pluginconfig WHERE enabled='t'" ) diff --git a/src/types.ts b/src/types.ts index 23851104..ebb33cb6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -14,6 +14,8 @@ export interface PluginsServerConfig { DISABLE_WEB: boolean WEB_PORT: number WEB_HOSTNAME: string + + __jestMock?: any } export interface PluginsServer extends PluginsServerConfig { From 84367710d96c00814c7caaa8fa9d006c6009ad16 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:10:18 +0100 Subject: [PATCH 20/61] less verbose logs in tests --- src/stats.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stats.ts b/src/stats.ts index b2dd9165..33e6d456 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -4,7 +4,8 @@ const historyIndex = new Map() export function logTime(name: string, time: number, error?: boolean): void { const ms = Math.round(time * 1000) / 1000 - console.log(`Running plugin ${name}: ${error ? 'ERROR IN ' : ''}${ms}ms`) + // TODO: add this back with better dev logging. Disabling since this trashes performance tests. + // console.log(`Running plugin ${name}: ${error ? 'ERROR IN ' : ''}${ms}ms`) if (!histories.has(name)) { histories.set(name, new Array(sampleCount)) historyIndex.set(name, -1) From de14d972645a6fda8b4681c9eb2db506e2921d2d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:10:39 +0100 Subject: [PATCH 21/61] add test to make sure adding more CPU cores makes plugins faster! --- src/worker/__tests__/piscina.test.ts | 64 +++++++++++++++++++--------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index 268b802c..4d7be9f4 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -2,7 +2,10 @@ import { defaultConfig } from '../../server' import { makePiscina } from '../piscina' import { PluginEvent } from 'posthog-plugins/src/types' import { performance } from 'perf_hooks' +import { mockJestWithIndex } from '../../__tests__/helpers/plugins' +import * as os from 'os' +jest.mock('../../sql') jest.setTimeout(300000) // 300 sec timeout function processOneEvent(processEvent: (event: PluginEvent) => Promise): Promise { @@ -10,7 +13,7 @@ function processOneEvent(processEvent: (event: PluginEvent) => Promise { - const piscina = makePiscina({ ...defaultConfig, WORKER_CONCURRENCY: 2 }) - - console.log('100 event warmup!') - await processCountEvents(100, piscina) +function setupPiscina(workers: number) { + return makePiscina({ + ...defaultConfig, + WORKER_CONCURRENCY: workers, + __jestMock: mockJestWithIndex(` + function processEvent (event, meta) { + let j = 0; for(let i = 0; i < 200000; i++) { j = i }; + event.properties = { "somewhere": "over the rainbow" }; + return event + } + `), + }) +} - console.log('--- START BENCHMARKING ---') - for (let i = 0; i < 10; i++) { - await processCountEvents(10000, piscina) - } +test('piscina 2-24 workers', async () => { + const cpuCount = os.cpus().length - await piscina.destroy() -}) + const workers = [1, 2, 4, 8, 16, 24, 32, 48, 64].filter((cores) => cores <= cpuCount) + const events = 10000 + const rounds = 5 -test('piscina 4 workers', async () => { - const piscina = makePiscina({ ...defaultConfig, WORKER_CONCURRENCY: 4 }) + const results: Record = {} + for (const cores of workers) { + const piscina = setupPiscina(cores) - console.log('100 event warmup!') - await processCountEvents(100, piscina) + // warmup + await processCountEvents(cpuCount * 4, piscina) - console.log('--- START BENCHMARKING ---') - for (let i = 0; i < 10; i++) { - await processCountEvents(10000, piscina) + // start + let throughput = 0 + for (let i = 0; i < 5; i++) { + const { eventsPerSecond } = await processCountEvents(10000, piscina) + throughput += eventsPerSecond + } + results[cores] = throughput / 5 + await piscina.destroy() } - await piscina.destroy() + console.log({ cpuCount }) + console.log(JSON.stringify(results, null, 2)) + + // expect that adding more cores (up to cpuCount) increases throughput + for (let i = 1; i < workers.length; i++) { + expect(results[workers[i - 1]]).toBeLessThan(results[workers[i]]) + } }) From 5d4685cbfdf3ae210c760801ac23953c4e711606 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:19:00 +0100 Subject: [PATCH 22/61] clone objects --- src/worker/worker.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/worker/worker.ts b/src/worker/worker.ts index 7a2c68b9..f914fcf5 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -18,7 +18,21 @@ export async function createWorker(config: PluginsServerConfig) { return `hello ${args[0]}!` } if (task === 'processEvent') { - return await runPlugins(server, args.event) + const processedEvent = await runPlugins(server, args.event) + // must clone the object, as we may get from VM2 something like { ..., properties: Proxy {} } + return cloneObject(processedEvent as Record) } } } + +function cloneObject(obj: Record) { + const clone: Record = {} + for (const i in obj) { + if (typeof obj[i] == 'object' && obj[i] !== null) { + clone[i] = cloneObject(obj[i]) + } else { + clone[i] = obj[i] + } + } + return clone +} From b96f37345a6cb38aa5001b587cfb82a276f09d52 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:19:42 +0100 Subject: [PATCH 23/61] add 12 --- src/worker/__tests__/piscina.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index 4d7be9f4..aece0496 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -62,7 +62,7 @@ function setupPiscina(workers: number) { test('piscina 2-24 workers', async () => { const cpuCount = os.cpus().length - const workers = [1, 2, 4, 8, 16, 24, 32, 48, 64].filter((cores) => cores <= cpuCount) + const workers = [1, 2, 4, 8, 12, 16, 24, 32, 48, 64].filter((cores) => cores <= cpuCount) const events = 10000 const rounds = 5 From e82f67fef58b4e118ffc9faea9f0b2ba46ed3a00 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:39:22 +0100 Subject: [PATCH 24/61] fix linter's worries --- .eslintrc.js | 1 + src/__tests__/helpers/plugins.ts | 10 ++++++++-- src/types.ts | 6 +++++- src/worker/__tests__/piscina.test.ts | 2 +- src/worker/worker.ts | 6 ++++-- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 4329f512..d16e41f2 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -8,6 +8,7 @@ module.exports = { '@typescript-eslint/explicit-function-return-type': 'off', '@typescript-eslint/no-non-null-assertion': 'off', '@typescript-eslint/no-var-requires': 'off', + '@typescript-eslint/no-explicit-any': 'off', curly: 'error', }, overrides: [ diff --git a/src/__tests__/helpers/plugins.ts b/src/__tests__/helpers/plugins.ts index 10ef4e6d..b41b71a4 100644 --- a/src/__tests__/helpers/plugins.ts +++ b/src/__tests__/helpers/plugins.ts @@ -81,12 +81,18 @@ function createZipBuffer(name: string, { indexJs, pluginJson }: { indexJs?: stri return zip.toBuffer() } -export const mockPluginWithArchive = (indexJs: string, pluginJson?: string) => ({ +export const mockPluginWithArchive = (indexJs: string, pluginJson?: string): Plugin => ({ ...plugin60, archive: createZipBuffer('posthog-maxmind-plugin', { indexJs, pluginJson }), }) -export const mockJestWithIndex = (indexJs: string) => ({ +export const mockJestWithIndex = ( + indexJs: string +): { + getPluginRows: Plugin[] + getPluginConfigRows: PluginConfig[] + getPluginAttachmentRows: PluginAttachmentDB[] +} => ({ getPluginRows: [mockPluginWithArchive(indexJs)], getPluginConfigRows: [pluginConfig39], getPluginAttachmentRows: [pluginAttachment1], diff --git a/src/types.ts b/src/types.ts index ebb33cb6..227b6dcc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -15,7 +15,11 @@ export interface PluginsServerConfig { WEB_PORT: number WEB_HOSTNAME: string - __jestMock?: any + __jestMock?: { + getPluginRows: Plugin[] + getPluginConfigRows: PluginConfig[] + getPluginAttachmentRows: PluginAttachmentDB[] + } } export interface PluginsServer extends PluginsServerConfig { diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index aece0496..cde60780 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -76,7 +76,7 @@ test('piscina 2-24 workers', async () => { // start let throughput = 0 for (let i = 0; i < 5; i++) { - const { eventsPerSecond } = await processCountEvents(10000, piscina) + const { eventsPerSecond } = await processCountEvents(10000, piscina) throughput += eventsPerSecond } results[cores] = throughput / 5 diff --git a/src/worker/worker.ts b/src/worker/worker.ts index f914fcf5..155bc222 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -2,7 +2,9 @@ import { runPlugins, setupPlugins } from '../plugins' import { createServer } from '../server' import { PluginsServerConfig } from '../types' -export async function createWorker(config: PluginsServerConfig) { +type TaskWorker = ({ task, args }: { task: string; args: any }) => Promise + +export async function createWorker(config: PluginsServerConfig): Promise { const [server, closeServer] = await createServer(config) await setupPlugins(server) @@ -13,7 +15,7 @@ export async function createWorker(config: PluginsServerConfig) { process.on(signal, closeJobs) } - return async ({ task, args }: { task: string; args: any }): Promise => { + return async ({ task, args }) => { if (task === 'hello') { return `hello ${args[0]}!` } From 6438aad88de16594b495b8343b7cfba817b3f24a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:43:27 +0100 Subject: [PATCH 25/61] remove extra word --- src/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql.ts b/src/sql.ts index 55911951..93cc1641 100644 --- a/src/sql.ts +++ b/src/sql.ts @@ -1,6 +1,6 @@ import { Plugin, PluginAttachmentDB, PluginConfig, PluginError, PluginsServer } from './types' -// This nice "mocking" system is used since we want to mock data in forked worker threads +// This nice "mocking" system is used since we want to mock data in worker threads. // Jest mocks don't penetrate that far. Improvements welcome. function areWeTestingWithJest() { return process.env.JEST_WORKER_ID !== undefined From 8f42315419fb346b45a784815a441899fe345cd9 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 16:44:49 +0100 Subject: [PATCH 26/61] improve display --- src/worker/__tests__/piscina.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index cde60780..a4c4c90c 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -75,11 +75,11 @@ test('piscina 2-24 workers', async () => { // start let throughput = 0 - for (let i = 0; i < 5; i++) { - const { eventsPerSecond } = await processCountEvents(10000, piscina) + for (let i = 0; i < rounds; i++) { + const { eventsPerSecond } = await processCountEvents(events, piscina) throughput += eventsPerSecond } - results[cores] = throughput / 5 + results[cores] = Math.round(throughput / rounds) await piscina.destroy() } From 889cbaf8a4d3e02c3e32c18d95702c6690429b2d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 17:18:29 +0100 Subject: [PATCH 27/61] remove the exact cpu count from the list --- src/worker/__tests__/piscina.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index a4c4c90c..5a66c0dd 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -62,7 +62,7 @@ function setupPiscina(workers: number) { test('piscina 2-24 workers', async () => { const cpuCount = os.cpus().length - const workers = [1, 2, 4, 8, 12, 16, 24, 32, 48, 64].filter((cores) => cores <= cpuCount) + const workers = [1, 2, 4, 8, 12, 16, 24, 32, 48, 64].filter((cores) => cores < cpuCount) const events = 10000 const rounds = 5 From ce933a755e75a7f79466fd79189ea475e3b9851f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 17:28:49 +0100 Subject: [PATCH 28/61] add postgres and redis services to github to make tests pass --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef34cefe..84e7b698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,26 @@ jobs: code-quality: name: Code quality and functionality checks runs-on: ubuntu-latest + + services: + postgres: + image: postgres:12 + env: + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis + ports: + # Maps port 6379 on service container to the host + # Needed because `redis` host is not discoverable for some reason + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v1 From 988608427a0c5a05b5fdb493757299c71585dc44 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 17:47:08 +0100 Subject: [PATCH 29/61] add ENV to config --- .github/workflows/ci.yml | 4 ++++ src/server.ts | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84e7b698..0eabc262 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,8 @@ jobs: postgres: image: postgres:12 env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: ['5432:5432'] options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 @@ -45,4 +47,6 @@ jobs: run: yarn lint - name: Test with Jest + env: + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres', run: yarn test diff --git a/src/server.ts b/src/server.ts index 6b6dcef1..1da986b2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -10,7 +10,17 @@ import { version } from '../package.json' import { PluginEvent } from 'posthog-plugins' import Piscina from 'piscina' -export const defaultConfig: PluginsServerConfig = { +function overrideWithEnv(config: PluginsServerConfig): PluginsServerConfig { + const newConfig: Record = { ...config } + for (const [key, value] of Object.entries(config)) { + if (process.env[key]) { + newConfig[key] = process.env[key] + } + } + return config +} + +export const defaultConfig: PluginsServerConfig = overrideWithEnv({ CELERY_DEFAULT_QUEUE: 'celery', DATABASE_URL: 'postgres://localhost:5432/posthog', PLUGINS_CELERY_QUEUE: 'posthog-plugins', @@ -21,7 +31,7 @@ export const defaultConfig: PluginsServerConfig = { WEB_PORT: 3008, WEB_HOSTNAME: '0.0.0.0', WORKER_CONCURRENCY: 0, // use all cores -} +}) export async function createServer( config: Partial = {} From 40195c1cdd749646e0169365e855b0bd1689cec5 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 4 Dec 2020 17:49:32 +0100 Subject: [PATCH 30/61] add ENV to config --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eabc262..10634185 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,5 +48,6 @@ jobs: - name: Test with Jest env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres', + DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + REDIS_URL: 'redis://localhost' run: yarn test From a5cbe9cb85954128b7059aab752c7e64254699fd Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 00:04:49 +0100 Subject: [PATCH 31/61] process events test wip --- src/worker/__tests__/piscina.test.ts | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index 5a66c0dd..c9d46a4e 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -22,10 +22,29 @@ function processOneEvent(processEvent: (event: PluginEvent) => Promise) { +function processOneBatch( + processEvents: (events: PluginEvent[]) => Promise, + batchSize: number = 10 +): Promise { + const events = [...Array(batchSize)].map(() => ({ + distinct_id: 'my_id', + ip: '127.0.0.1', + site_url: 'http://localhost', + team_id: 2, + now: new Date().toISOString(), + event: 'default event', + properties: { key: 'value' }, + })) + + return processEvents(events) +} + +async function processCountEvents(count: number, piscina: ReturnType, batched: false) { const startTime = performance.now() const promises = Array(count) const processEvent = (event: PluginEvent) => piscina.runTask({ task: 'processEvent', args: { event } }) + const processEvents = (events: PluginEvent[]) => piscina.runTask({ task: 'processEvents', args: { events } }) + for (let i = 0; i < count; i++) { promises[i] = processOneEvent(processEvent) } @@ -46,6 +65,8 @@ async function processCountEvents(count: number, piscina: ReturnType Date: Sat, 5 Dec 2020 11:55:47 +0100 Subject: [PATCH 32/61] add db:init script --- package.json | 3 +- tasks/db-init.ts | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 tasks/db-init.ts diff --git a/package.json b/package.json index 1cb0ff11..2928dc01 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "lint:fix": "eslint --fix .", "prettier": "prettier --write .", "prettier:check": "prettier --check .", - "prepublish": "yarn test && yarn build" + "prepublish": "yarn test && yarn build", + "task:db-init": "ts-node ./tasks/db-init.ts" }, "bin": { "posthog-plugin-server": "./bin/posthog-plugin-server" diff --git a/tasks/db-init.ts b/tasks/db-init.ts new file mode 100644 index 00000000..af483f28 --- /dev/null +++ b/tasks/db-init.ts @@ -0,0 +1,88 @@ +import { createServer } from '../src/server' + +async function task() { + const [server, closeServer] = await createServer() + + await server.db.query(createPlugin) + await server.db.query(createPluginAttachment) + await server.db.query(createPluginConfig) + + await closeServer() +} + +const ifNotExists = 'if not exists' + +const createPlugin = ` + create table ${ifNotExists} posthog_plugin + ( + id serial not null + constraint posthog_plugin_pkey + primary key, + name varchar(200), + description text, + url varchar(800), + config_schema jsonb not null, + tag varchar(200), + archive bytea, + from_json boolean not null, + from_web boolean not null, + error jsonb + ); +` + +const createPluginAttachment = ` + create table ${ifNotExists} posthog_pluginattachment + ( + id serial not null + constraint posthog_pluginattachment_pkey + primary key, + key varchar(200) not null, + content_type varchar(200) not null, + file_name varchar(200) not null, + file_size integer not null, + contents bytea not null, + plugin_config_id integer not null + constraint posthog_pluginattach_plugin_config_id_cc94a1b9_fk_posthog_p + references posthog_pluginconfig + deferrable initially deferred, + team_id integer + constraint posthog_pluginattachment_team_id_415eacc7_fk_posthog_team_id + references posthog_team + deferrable initially deferred + ); + + create index ${ifNotExists} posthog_pluginattachment_plugin_config_id_cc94a1b9 + on posthog_pluginattachment (plugin_config_id); + + create index ${ifNotExists} posthog_pluginattachment_team_id_415eacc7 + on posthog_pluginattachment (team_id); +` + +const createPluginConfig = ` + create table ${ifNotExists} posthog_pluginconfig + ( + id serial not null + constraint posthog_pluginconfig_pkey + primary key, + team_id integer + constraint posthog_pluginconfig_team_id_71185766_fk_posthog_team_id + references posthog_team + deferrable initially deferred, + plugin_id integer not null + constraint posthog_pluginconfig_plugin_id_d014ca1c_fk_posthog_plugin_id + references posthog_plugin + deferrable initially deferred, + enabled boolean not null, + "order" integer, + config jsonb not null, + error jsonb + ); + + create index ${ifNotExists} posthog_pluginconfig_team_id_71185766 + on posthog_pluginconfig (team_id); + + create index ${ifNotExists} posthog_pluginconfig_plugin_id_d014ca1c + on posthog_pluginconfig (plugin_id); +` + +task() From 63a22ce520ae38f365458229b3cac36be924806f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 11:56:34 +0100 Subject: [PATCH 33/61] add db:init script to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10634185..43dd50f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,4 +50,4 @@ jobs: env: DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' REDIS_URL: 'redis://localhost' - run: yarn test + run: yarn task:db-init && yarn test From 9ae210a9eaa0b7c231db1a0e51fe34f3e4447a3a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 11:55:47 +0100 Subject: [PATCH 34/61] add db:init script --- package.json | 3 +- tasks/db-init.ts | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 tasks/db-init.ts diff --git a/package.json b/package.json index 1cb0ff11..2928dc01 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "lint:fix": "eslint --fix .", "prettier": "prettier --write .", "prettier:check": "prettier --check .", - "prepublish": "yarn test && yarn build" + "prepublish": "yarn test && yarn build", + "task:db-init": "ts-node ./tasks/db-init.ts" }, "bin": { "posthog-plugin-server": "./bin/posthog-plugin-server" diff --git a/tasks/db-init.ts b/tasks/db-init.ts new file mode 100644 index 00000000..af483f28 --- /dev/null +++ b/tasks/db-init.ts @@ -0,0 +1,88 @@ +import { createServer } from '../src/server' + +async function task() { + const [server, closeServer] = await createServer() + + await server.db.query(createPlugin) + await server.db.query(createPluginAttachment) + await server.db.query(createPluginConfig) + + await closeServer() +} + +const ifNotExists = 'if not exists' + +const createPlugin = ` + create table ${ifNotExists} posthog_plugin + ( + id serial not null + constraint posthog_plugin_pkey + primary key, + name varchar(200), + description text, + url varchar(800), + config_schema jsonb not null, + tag varchar(200), + archive bytea, + from_json boolean not null, + from_web boolean not null, + error jsonb + ); +` + +const createPluginAttachment = ` + create table ${ifNotExists} posthog_pluginattachment + ( + id serial not null + constraint posthog_pluginattachment_pkey + primary key, + key varchar(200) not null, + content_type varchar(200) not null, + file_name varchar(200) not null, + file_size integer not null, + contents bytea not null, + plugin_config_id integer not null + constraint posthog_pluginattach_plugin_config_id_cc94a1b9_fk_posthog_p + references posthog_pluginconfig + deferrable initially deferred, + team_id integer + constraint posthog_pluginattachment_team_id_415eacc7_fk_posthog_team_id + references posthog_team + deferrable initially deferred + ); + + create index ${ifNotExists} posthog_pluginattachment_plugin_config_id_cc94a1b9 + on posthog_pluginattachment (plugin_config_id); + + create index ${ifNotExists} posthog_pluginattachment_team_id_415eacc7 + on posthog_pluginattachment (team_id); +` + +const createPluginConfig = ` + create table ${ifNotExists} posthog_pluginconfig + ( + id serial not null + constraint posthog_pluginconfig_pkey + primary key, + team_id integer + constraint posthog_pluginconfig_team_id_71185766_fk_posthog_team_id + references posthog_team + deferrable initially deferred, + plugin_id integer not null + constraint posthog_pluginconfig_plugin_id_d014ca1c_fk_posthog_plugin_id + references posthog_plugin + deferrable initially deferred, + enabled boolean not null, + "order" integer, + config jsonb not null, + error jsonb + ); + + create index ${ifNotExists} posthog_pluginconfig_team_id_71185766 + on posthog_pluginconfig (team_id); + + create index ${ifNotExists} posthog_pluginconfig_plugin_id_d014ca1c + on posthog_pluginconfig (plugin_id); +` + +task() From 629751f15715026098200e00ebf68a1b8f640c68 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 11:56:34 +0100 Subject: [PATCH 35/61] add db:init script to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10634185..43dd50f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,4 +50,4 @@ jobs: env: DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' REDIS_URL: 'redis://localhost' - run: yarn test + run: yarn task:db-init && yarn test From fe0c6cb6e72135cdcdd1cb2fef90486d2a74d876 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 13:23:37 +0100 Subject: [PATCH 36/61] No prepublish on yarn install --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2928dc01..aec6076e 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "lint:fix": "eslint --fix .", "prettier": "prettier --write .", "prettier:check": "prettier --check .", - "prepublish": "yarn test && yarn build", + "prepublishOnly": "yarn test && yarn build", "task:db-init": "ts-node ./tasks/db-init.ts" }, "bin": { From 43bd0e31f0b61933a1aab5a50af91f592e9adc07 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 15:58:36 +0100 Subject: [PATCH 37/61] processEvents in vm --- src/__tests__/vm.test.ts | 49 ++++++++++++++++++++++++++++++++++++++-- src/types.ts | 1 + src/vm.ts | 15 +++++++++++- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/__tests__/vm.test.ts b/src/__tests__/vm.test.ts index 34d57041..33b7276e 100644 --- a/src/__tests__/vm.test.ts +++ b/src/__tests__/vm.test.ts @@ -55,8 +55,9 @@ test('empty plugins', async () => { const vm = createPluginConfigVM(mockServer, mockConfig, indexJs, libJs) expect(Object.keys(vm).sort()).toEqual(['methods', 'vm']) - expect(Object.keys(vm.methods).sort()).toEqual(['processEvent']) + expect(Object.keys(vm.methods).sort()).toEqual(['processEvent', 'processEvents']) expect(vm.methods.processEvent).toEqual(undefined) + expect(vm.methods.processEvents).toEqual(undefined) }) test('processEvent', async () => { @@ -68,17 +69,61 @@ test('processEvent', async () => { ` const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) expect(vm.methods.processEvent).not.toEqual(undefined) + expect(vm.methods.processEvents).not.toEqual(undefined) const event: PluginEvent = { ...defaultEvent, event: 'original event', } - const newEvent = await vm.methods.processEvent(event) + expect(event.event).toEqual('changed event') + expect(newEvent.event).toEqual('changed event') + expect(newEvent).toBe(event) + + const events: PluginEvent[] = [ + { + ...defaultEvent, + event: 'original event', + }, + ] + const newEvents = await vm.methods.processEvents(events) + expect(events[0].event).toEqual('changed event') + expect(newEvents[0].event).toEqual('changed event') + expect(newEvents[0]).toBe(events[0]) +}) + +test('processEvents', async () => { + const indexJs = ` + function processEvents (events, meta) { + return events.map(event => { + event.event = 'changed event' + return event + }) + } + ` + const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) + expect(vm.methods.processEvent).not.toEqual(undefined) + expect(vm.methods.processEvents).not.toEqual(undefined) + const event: PluginEvent = { + ...defaultEvent, + event: 'original event', + } + const newEvent = await vm.methods.processEvent(event) expect(event.event).toEqual('changed event') expect(newEvent.event).toEqual('changed event') expect(newEvent).toBe(event) + + const events: PluginEvent[] = [ + { + ...defaultEvent, + event: 'original event', + }, + ] + const newEvents = await vm.methods.processEvents(events) + expect(events[0].event).toEqual('changed event') + expect(newEvents[0].event).toEqual('changed event') + expect(newEvents[0]).toBe(events[0]) }) test('processEvent without returning', async () => { diff --git a/src/types.ts b/src/types.ts index 227b6dcc..29938a86 100644 --- a/src/types.ts +++ b/src/types.ts @@ -101,5 +101,6 @@ export interface PluginConfigVMReponse { vm: VM methods: { processEvent: (event: PluginEvent) => Promise + processEvents: (events: PluginEvent[]) => Promise } } diff --git a/src/vm.ts b/src/vm.ts index bbc0951c..99c06e1b 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -58,9 +58,22 @@ export function createPluginConfigVM( // run the plugin setup script, if present __callWithMeta('setupPlugin'); + // we have processEvent, but not processEvents + if (!__getExported('processEvents') && __getExported('processEvent')) { + function processEvents (events, meta) { + return events.map(event => processEvent(event, meta)).filter(e => e) + } + // we have processEvents, but not processEvent + } else if (!__getExported('processEvent') && __getExported('processEvents')) { + function processEvent (event, meta) { + return processEvents([event], meta)?.[0] + } + } + // export various functions const __methods = { - processEvent: __bindMeta('processEvent') + processEvent: __bindMeta('processEvent'), + processEvents: __bindMeta('processEvents') }; ` ) From 2b24cb6252f6297f0a251ec3211c0613f69ef3f9 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 16:46:17 +0100 Subject: [PATCH 38/61] run plugins on batches of events --- src/__tests__/vm.test.ts | 12 ++++++--- src/plugins.ts | 53 +++++++++++++++++++++++++++++++++++++++- src/vm.ts | 11 ++++++--- src/worker/worker.ts | 7 +++++- 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/src/__tests__/vm.test.ts b/src/__tests__/vm.test.ts index 33b7276e..f8103f99 100644 --- a/src/__tests__/vm.test.ts +++ b/src/__tests__/vm.test.ts @@ -406,7 +406,9 @@ test('prepareForRun without token', async () => { event: 'prepareForRun event', properties: {}, } - const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) + const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) as ( + event: PluginEvent + ) => Promise expect(processEvent).toBeDefined() @@ -434,7 +436,9 @@ test('prepareForRun with token gets posthog', async () => { token: 'posthog-token', }, } - const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) + const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) as ( + event: PluginEvent + ) => Promise expect(processEvent).toBeDefined() await processEvent!(event) @@ -461,7 +465,9 @@ test('posthog.capture', async () => { token: 'posthog-token', }, } - const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) + const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) as ( + event: PluginEvent + ) => Promise expect(processEvent).toBeDefined() await processEvent!(event) diff --git a/src/plugins.ts b/src/plugins.ts index 94585670..bf729809 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -186,7 +186,9 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro for (const pluginConfig of pluginsToRun.reverse()) { if (pluginConfig.vm) { - const processEvent = prepareForRun(server, event.team_id, pluginConfig, 'processEvent', event) + const processEvent = prepareForRun(server, event.team_id, pluginConfig, 'processEvent', event) as ( + event: PluginEvent + ) => Promise if (processEvent) { const startTime = performance.now() @@ -209,3 +211,52 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro return returnedEvent } + +export async function runPluginsBatch(server: PluginsServer, events: PluginEvent[]): Promise { + const eventsByTeam = new Map() + // TODO: batch also by site_url and event.properties.token + for (const event of events) { + if (eventsByTeam.has(event.team_id)) { + eventsByTeam.get(event.team_id)!.push(event) + } else { + eventsByTeam.set(event.team_id, [event]) + } + } + + let allReturnedEvents: PluginEvent[] = [] + + for (const [teamId, teamEvents] of eventsByTeam.entries()) { + const pluginsToRun = server.pluginConfigsPerTeam.get(teamId) || server.defaultConfigs + + let returnedEvents: PluginEvent[] = teamEvents + + for (const pluginConfig of pluginsToRun.reverse()) { + if (pluginConfig.vm && returnedEvents.length > 0) { + const processEvents = prepareForRun(server, teamId, pluginConfig, 'processEvents', events[0]) as ( + events: PluginEvent[] + ) => Promise + + if (processEvents) { + const startTime = performance.now() + try { + returnedEvents = (await processEvents(returnedEvents)) || null + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms) + } catch (error) { + await processError(server, pluginConfig, error, returnedEvents[0]) + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms, true) + } + } + + if (!returnedEvents) { + returnedEvents = [] + } + } + } + + allReturnedEvents = allReturnedEvents.concat(returnedEvents) + } + + return allReturnedEvents +} diff --git a/src/vm.ts b/src/vm.ts index 99c06e1b..11de9186 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -88,9 +88,13 @@ export function prepareForRun( server: PluginsServer, teamId: number, pluginConfig: PluginConfig, // might have team_id=0 - method: 'processEvent', + method: 'processEvent' | 'processEvents', event?: PluginEvent -): null | ((event: PluginEvent) => Promise) | (() => Promise) { +): + | null + | ((event: PluginEvent) => Promise) + | ((events: PluginEvent[]) => Promise) + | (() => Promise) { if (!pluginConfig.vm?.methods[method]) { return null } @@ -98,7 +102,8 @@ export function prepareForRun( const { vm } = pluginConfig.vm if (event?.properties?.token) { - // TODO: this should be nicer... and it's not optimised for batch processing + // TODO: this should be nicer... and it's not optimised for batch processing! + // We should further split the batches per site_url and token! const posthog = createInternalPostHogInstance( event.properties.token, { apiHost: event.site_url, fetch }, diff --git a/src/worker/worker.ts b/src/worker/worker.ts index 155bc222..2ffdf50e 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -1,4 +1,4 @@ -import { runPlugins, setupPlugins } from '../plugins' +import { runPlugins, runPluginsBatch, setupPlugins } from '../plugins' import { createServer } from '../server' import { PluginsServerConfig } from '../types' @@ -24,6 +24,11 @@ export async function createWorker(config: PluginsServerConfig): Promise) } + if (task === 'processEvents') { + const processedEvents = await runPluginsBatch(server, args.events) + // must clone the object, as we may get from VM2 something like { ..., properties: Proxy {} } + return cloneObject(processedEvents as any[]) + } } } From 9b3c82d4f767be66a4ec1f23e7ee7e22f8e87955 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sat, 5 Dec 2020 16:46:24 +0100 Subject: [PATCH 39/61] tests for batching performance --- src/worker/__tests__/piscina.test.ts | 96 ++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 12 deletions(-) diff --git a/src/worker/__tests__/piscina.test.ts b/src/worker/__tests__/piscina.test.ts index c9d46a4e..a4141572 100644 --- a/src/worker/__tests__/piscina.test.ts +++ b/src/worker/__tests__/piscina.test.ts @@ -8,7 +8,7 @@ import * as os from 'os' jest.mock('../../sql') jest.setTimeout(300000) // 300 sec timeout -function processOneEvent(processEvent: (event: PluginEvent) => Promise): Promise { +function processOneEvent(processEvent: (event: PluginEvent) => Promise, index: number): Promise { const defaultEvent = { distinct_id: 'my_id', ip: '127.0.0.1', @@ -16,7 +16,7 @@ function processOneEvent(processEvent: (event: PluginEvent) => Promise Promise Promise, - batchSize: number = 10 + batchSize: number, + batchIndex: number ): Promise { - const events = [...Array(batchSize)].map(() => ({ + const events = [...Array(batchSize)].map((_, i) => ({ distinct_id: 'my_id', ip: '127.0.0.1', site_url: 'http://localhost', team_id: 2, now: new Date().toISOString(), event: 'default event', - properties: { key: 'value' }, + properties: { key: 'value', batchIndex, indexInBatch: i }, })) return processEvents(events) } -async function processCountEvents(count: number, piscina: ReturnType, batched: false) { +async function processCountEvents(piscina: ReturnType, count: number, batchSize = 1) { const startTime = performance.now() const promises = Array(count) + const processEvent = (event: PluginEvent) => piscina.runTask({ task: 'processEvent', args: { event } }) const processEvents = (events: PluginEvent[]) => piscina.runTask({ task: 'processEvents', args: { events } }) for (let i = 0; i < count; i++) { - promises[i] = processOneEvent(processEvent) + promises[i] = batchSize === 1 ? processOneEvent(processEvent, i) : processOneBatch(processEvents, batchSize, i) } // this will get heavy for tests > 10k events, should chunk them somehow... await Promise.all(promises) @@ -54,7 +56,7 @@ async function processCountEvents(count: number, piscina: ReturnType processEvent(event, meta)) + } `), }) } @@ -92,12 +96,80 @@ test('piscina 2-24 workers', async () => { const piscina = setupPiscina(cores) // warmup - await processCountEvents(cpuCount * 4, piscina) + await processCountEvents(piscina, cpuCount * 4, 1) + + // start + let throughput = 0 + for (let i = 0; i < rounds; i++) { + const { eventsPerSecond } = await processCountEvents(piscina, events, 1) + throughput += eventsPerSecond + } + results[cores] = Math.round(throughput / rounds) + await piscina.destroy() + } + + console.log({ cpuCount }) + console.log(JSON.stringify(results, null, 2)) + + // expect that adding more cores (up to cpuCount) increases throughput + for (let i = 1; i < workers.length; i++) { + expect(results[workers[i - 1]]).toBeLessThan(results[workers[i]]) + } +}) + +test('piscina 2-24 workers batch of 10', async () => { + const cpuCount = os.cpus().length + + const workers = [1, 2, 4, 8, 12, 16, 24, 32, 48, 64].filter((cores) => cores < cpuCount) + const events = 1000 + const batchSize = 10 + const rounds = 5 + + const results: Record = {} + for (const cores of workers) { + const piscina = setupPiscina(cores) + + // warmup + await processCountEvents(piscina, cpuCount * 4, batchSize) + + // start + let throughput = 0 + for (let i = 0; i < rounds; i++) { + const { eventsPerSecond } = await processCountEvents(piscina, events, batchSize) + throughput += eventsPerSecond + } + results[cores] = Math.round(throughput / rounds) + await piscina.destroy() + } + + console.log({ cpuCount }) + console.log(JSON.stringify(results, null, 2)) + + // expect that adding more cores (up to cpuCount) increases throughput + for (let i = 1; i < workers.length; i++) { + expect(results[workers[i - 1]]).toBeLessThan(results[workers[i]]) + } +}) + +test('piscina 2-24 workers batch of 100', async () => { + const cpuCount = os.cpus().length + + const workers = [1, 2, 4, 8, 12, 16, 24, 32, 48, 64].filter((cores) => cores < cpuCount) + const events = 100 + const batchSize = 100 + const rounds = 5 + + const results: Record = {} + for (const cores of workers) { + const piscina = setupPiscina(cores) + + // warmup + await processCountEvents(piscina, cpuCount * 4, batchSize) // start let throughput = 0 for (let i = 0; i < rounds; i++) { - const { eventsPerSecond } = await processCountEvents(events, piscina) + const { eventsPerSecond } = await processCountEvents(piscina, events, batchSize) throughput += eventsPerSecond } results[cores] = Math.round(throughput / rounds) From 8b8f22e291a13f5f7135f6c435b9ac010118ea6f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Sun, 6 Dec 2020 01:14:09 +0100 Subject: [PATCH 40/61] async processEvent & events --- src/__tests__/vm.test.ts | 104 +++++++++++++++++++++++++++++++++++++++ src/vm.ts | 11 +++-- 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/src/__tests__/vm.test.ts b/src/__tests__/vm.test.ts index f8103f99..3cc7d42f 100644 --- a/src/__tests__/vm.test.ts +++ b/src/__tests__/vm.test.ts @@ -92,6 +92,38 @@ test('processEvent', async () => { expect(newEvents[0]).toBe(events[0]) }) +test('async processEvent', async () => { + const indexJs = ` + async function processEvent (event, meta) { + event.event = 'changed event' + return event + } + ` + const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) + expect(vm.methods.processEvent).not.toEqual(undefined) + expect(vm.methods.processEvents).not.toEqual(undefined) + + const event: PluginEvent = { + ...defaultEvent, + event: 'original event', + } + const newEvent = await vm.methods.processEvent(event) + expect(event.event).toEqual('changed event') + expect(newEvent.event).toEqual('changed event') + expect(newEvent).toBe(event) + + const events: PluginEvent[] = [ + { + ...defaultEvent, + event: 'original event', + }, + ] + const newEvents = await vm.methods.processEvents(events) + expect(events[0].event).toEqual('changed event') + expect(newEvents[0].event).toEqual('changed event') + expect(newEvents[0]).toBe(events[0]) +}) + test('processEvents', async () => { const indexJs = ` function processEvents (events, meta) { @@ -126,6 +158,78 @@ test('processEvents', async () => { expect(newEvents[0]).toBe(events[0]) }) +test('async processEvents', async () => { + const indexJs = ` + async function processEvents (events, meta) { + return events.map(event => { + event.event = 'changed event' + return event + }) + } + ` + const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) + expect(vm.methods.processEvent).not.toEqual(undefined) + expect(vm.methods.processEvents).not.toEqual(undefined) + + const event: PluginEvent = { + ...defaultEvent, + event: 'original event', + } + const newEvent = await vm.methods.processEvent(event) + expect(event.event).toEqual('changed event') + expect(newEvent.event).toEqual('changed event') + expect(newEvent).toBe(event) + + const events: PluginEvent[] = [ + { + ...defaultEvent, + event: 'original event', + }, + ] + const newEvents = await vm.methods.processEvents(events) + expect(events[0].event).toEqual('changed event') + expect(newEvents[0].event).toEqual('changed event') + expect(newEvents[0]).toBe(events[0]) +}) + +test('processEvent && processEvents', async () => { + const indexJs = ` + function processEvent (event, meta) { + event.event = 'changed event 1' + return event + } + function processEvents (events, meta) { + return events.map(event => { + event.event = 'changed event 2' + return event + }) + } + ` + const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) + expect(vm.methods.processEvent).not.toEqual(undefined) + expect(vm.methods.processEvents).not.toEqual(undefined) + + const event: PluginEvent = { + ...defaultEvent, + event: 'original event', + } + const newEvent = await vm.methods.processEvent(event) + expect(event.event).toEqual('changed event 1') + expect(newEvent.event).toEqual('changed event 1') + expect(newEvent).toBe(event) + + const events: PluginEvent[] = [ + { + ...defaultEvent, + event: 'original event', + }, + ] + const newEvents = await vm.methods.processEvents(events) + expect(events[0].event).toEqual('changed event 2') + expect(newEvents[0].event).toEqual('changed event 2') + expect(newEvents[0]).toBe(events[0]) +}) + test('processEvent without returning', async () => { const indexJs = ` function processEvent (event, meta) { diff --git a/src/vm.ts b/src/vm.ts index 11de9186..15c7482c 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -60,13 +60,16 @@ export function createPluginConfigVM( // we have processEvent, but not processEvents if (!__getExported('processEvents') && __getExported('processEvent')) { - function processEvents (events, meta) { - return events.map(event => processEvent(event, meta)).filter(e => e) + exports.processEvents = async function processEvents (events, meta) { + const processEvent = __getExported('processEvent'); + const pArray = events.map(async event => await processEvent(event, meta)) + const response = await Promise.all(pArray); + return response.filter(r => r) } // we have processEvents, but not processEvent } else if (!__getExported('processEvent') && __getExported('processEvents')) { - function processEvent (event, meta) { - return processEvents([event], meta)?.[0] + exports.processEvent = async function processEvent (event, meta) { + return (await (__getExported('processEvents'))([event], meta))?.[0] } } From 31a264d1b87152d8b1760b6378ece4a90fba591e Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 09:57:39 +0100 Subject: [PATCH 41/61] lint --- tests/piscina.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/piscina.test.ts b/tests/piscina.test.ts index 0b693942..9d4c109a 100644 --- a/tests/piscina.test.ts +++ b/tests/piscina.test.ts @@ -8,7 +8,10 @@ import * as os from 'os' jest.mock('../src/sql') jest.setTimeout(300000) // 300 sec timeout -function processOneEvent(processEvent: (event: PluginEvent) => Promise, index: number): Promise { +function processOneEvent( + processEvent: (event: PluginEvent) => Promise, + index: number +): Promise { const defaultEvent = { distinct_id: 'my_id', ip: '127.0.0.1', From 0f8d7f65346b09af428b4a229b95ffad3309af33 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 14:16:56 +0100 Subject: [PATCH 42/61] performance test batches --- tests/worker.test.ts | 60 ++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/tests/worker.test.ts b/tests/worker.test.ts index 8495a399..a255a6e0 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -46,7 +46,6 @@ function processOneBatch( async function processCountEvents(piscina: ReturnType, count: number, batchSize = 1) { const maxPromises = 1000 - const startTime = performance.now() const promises = Array(maxPromises) const processEvent = (event: PluginEvent) => piscina.runTask({ task: 'processEvent', args: { event } }) const processEvents = (events: PluginEvent[]) => piscina.runTask({ task: 'processEvents', args: { events } }) @@ -60,18 +59,6 @@ async function processCountEvents(piscina: ReturnType, count } await Promise.all(promises) } - - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - - const log = { - eventsPerSecond: 1000 / (ms / count), - events: count, - concurrency: piscina.threads.length, - totalMs: ms, - averageEventMs: ms / count, - } - - return log } function setupPiscina(workers: number, code: string, tasksPerWorker: number) { @@ -90,9 +77,9 @@ test('piscina worker test', async () => { const coreCount = os.cpus().length const workerThreads = [1, 2, 4, 8, 12, 16].filter((threads) => - isDevRun ? threads < coreCount : threads <= coreCount + isDevRun ? threads <= 8 : threads <= coreCount ) - const rounds = 5 + const rounds = isDevRun ? 1 : 3 const tests: { testName: string; events: number; testCode: string }[] = [ { @@ -130,28 +117,35 @@ test('piscina worker test', async () => { ] const results: Array> = [] - for (const { testName, events, testCode } of tests) { - const result: Record = { - testName, - coreCount, - } - for (const threads of workerThreads) { - const piscina = setupPiscina(threads, testCode, 100) + for (const { testName, events: _events, testCode } of tests) { + const events = isDevRun ? _events / 10 : _events + for (const batchSize of [1, 10, 100].filter((size) => size <= events)) { + const result: Record = { + testName, + coreCount, + events, + batchSize, + } + for (const threads of workerThreads) { + const piscina = setupPiscina(threads, testCode, 100) - // warmup - await processCountEvents(piscina, threads * 4) + // warmup + await processCountEvents(piscina, threads * 4) + + // start + const startTime = performance.now() + for (let i = 0; i < rounds; i++) { + await processCountEvents(piscina, events / batchSize, batchSize) + } + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + const throughput = Math.round(1000 / (ms / events / rounds)) + result[`${threads} thread${threads === 1 ? '' : 's'}`] = Math.round(throughput) - // start - let throughput = 0 - for (let i = 0; i < rounds; i++) { - const { eventsPerSecond } = await processCountEvents(piscina, isDevRun ? events / 10 : events) - throughput += eventsPerSecond + await piscina.destroy() } - result[`${threads} threads`] = Math.round(throughput / rounds) - await piscina.destroy() + results.push(result) + console.log(JSON.stringify({ result }, null, 2)) } - results.push(result) - console.log(JSON.stringify({ result }, null, 2)) } console.table(results) }) From 2225f6f601ce1092db6b51b96595757b98e7201d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 14:17:14 +0100 Subject: [PATCH 43/61] not a dev run --- tests/worker.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/worker.test.ts b/tests/worker.test.ts index a255a6e0..110694c6 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -73,7 +73,7 @@ function setupPiscina(workers: number, code: string, tasksPerWorker: number) { test('piscina worker test', async () => { // Uncomment this to become a 10x developer and make the test run just as fast! - const isDevRun = true + const isDevRun = false const coreCount = os.cpus().length const workerThreads = [1, 2, 4, 8, 12, 16].filter((threads) => From 42920e4a2c85ec18c99c5db6be9313982737bd5a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 14:26:44 +0100 Subject: [PATCH 44/61] prettier for ts --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 04330d56..6b59d786 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,6 @@ } }, "lint-staged": { - "*.{js,css,scss}": "prettier --write" + "*.{js,css,scss,ts}": "prettier --write" } } From 3038e1e32b0016ed18805cff2c432db32cb6d8f5 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 14:26:59 +0100 Subject: [PATCH 45/61] prettier --- tests/worker.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/worker.test.ts b/tests/worker.test.ts index 110694c6..9b779e00 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -76,9 +76,7 @@ test('piscina worker test', async () => { const isDevRun = false const coreCount = os.cpus().length - const workerThreads = [1, 2, 4, 8, 12, 16].filter((threads) => - isDevRun ? threads <= 8 : threads <= coreCount - ) + const workerThreads = [1, 2, 4, 8, 12, 16].filter((threads) => (isDevRun ? threads <= 8 : threads <= coreCount)) const rounds = isDevRun ? 1 : 3 const tests: { testName: string; events: number; testCode: string }[] = [ @@ -105,7 +103,7 @@ test('piscina worker test', async () => { }, { testName: 'timeout100ms', - events: 2000, + events: 10000, testCode: ` async function processEvent (event, meta) { await new Promise(resolve => __jestSetTimeout(() => resolve(), 100)) From 9ffb68ffd06517d9797088ac6ddf54f3744060b3 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 14:53:25 +0100 Subject: [PATCH 46/61] remove support for PostHog.capture inside `processEvent` for now. This will go back inside the scheduled/webhook plugins --- src/plugins.ts | 64 ++++++++++++----------------- src/vm.ts | 35 ---------------- src/worker/worker.ts | 4 +- tests/plugins.test.ts | 2 +- tests/vm.test.ts | 95 +------------------------------------------ 5 files changed, 29 insertions(+), 171 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index a64bc858..a3d42858 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -1,6 +1,6 @@ import * as path from 'path' import * as fs from 'fs' -import { createPluginConfigVM, prepareForRun } from './vm' +import { createPluginConfigVM } from './vm' import { PluginsServer, PluginConfig, PluginJsonConfig, TeamId } from './types' import { PluginEvent, PluginAttachment } from 'posthog-plugins' import { clearError, processError } from './error' @@ -185,22 +185,17 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro let returnedEvent: PluginEvent | null = event for (const pluginConfig of pluginsToRun.reverse()) { - if (pluginConfig.vm) { - const processEvent = prepareForRun(server, event.team_id, pluginConfig, 'processEvent', event) as ( - event: PluginEvent - ) => Promise - - if (processEvent) { - const startTime = performance.now() - try { - returnedEvent = (await processEvent(returnedEvent)) || null - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms) - } catch (error) { - await processError(server, pluginConfig, error, returnedEvent) - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms, true) - } + if (pluginConfig.vm?.methods?.processEvent) { + const { processEvent } = pluginConfig.vm.methods + const startTime = performance.now() + try { + returnedEvent = (await processEvent(returnedEvent)) || null + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms) + } catch (error) { + await processError(server, pluginConfig, error, returnedEvent) + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms, true) } if (!returnedEvent) { @@ -212,9 +207,9 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro return returnedEvent } -export async function runPluginsBatch(server: PluginsServer, events: PluginEvent[]): Promise { +export async function runPluginsOnBatch(server: PluginsServer, events: PluginEvent[]): Promise { const eventsByTeam = new Map() - // TODO: batch also by site_url and event.properties.token + for (const event of events) { if (eventsByTeam.has(event.team_id)) { eventsByTeam.get(event.team_id)!.push(event) @@ -231,26 +226,17 @@ export async function runPluginsBatch(server: PluginsServer, events: PluginEvent let returnedEvents: PluginEvent[] = teamEvents for (const pluginConfig of pluginsToRun.reverse()) { - if (pluginConfig.vm && returnedEvents.length > 0) { - const processEvents = prepareForRun(server, teamId, pluginConfig, 'processEvents', events[0]) as ( - events: PluginEvent[] - ) => Promise - - if (processEvents) { - const startTime = performance.now() - try { - returnedEvents = (await processEvents(returnedEvents)) || null - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms) - } catch (error) { - await processError(server, pluginConfig, error, returnedEvents[0]) - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms, true) - } - } - - if (!returnedEvents) { - returnedEvents = [] + if (pluginConfig.vm?.methods?.processEvents && returnedEvents.length > 0) { + const { processEvents } = pluginConfig.vm.methods + const startTime = performance.now() + try { + returnedEvents = (await processEvents(returnedEvents)) || [] + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms) + } catch (error) { + await processError(server, pluginConfig, error, returnedEvents[0]) + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms, true) } } } diff --git a/src/vm.ts b/src/vm.ts index 272856d6..f8e2e9ba 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -93,38 +93,3 @@ export function createPluginConfigVM( methods: vm.run('__methods'), } } - -export function prepareForRun( - server: PluginsServer, - teamId: number, - pluginConfig: PluginConfig, // might have team_id=0 - method: 'processEvent' | 'processEvents', - event?: PluginEvent -): - | null - | ((event: PluginEvent) => Promise) - | ((events: PluginEvent[]) => Promise) - | (() => Promise) { - if (!pluginConfig.vm?.methods[method]) { - return null - } - - const { vm } = pluginConfig.vm - - if (event?.properties?.token) { - // TODO: this should be nicer... and it's not optimised for batch processing! - // We should further split the batches per site_url and token! - const posthog = createInternalPostHogInstance( - event.properties.token, - { apiHost: event.site_url, fetch }, - { - performance: performance, - } - ) - vm.freeze(posthog, 'posthog') - } else { - vm.freeze(null, 'posthog') - } - - return pluginConfig.vm.methods[method] -} diff --git a/src/worker/worker.ts b/src/worker/worker.ts index 6abdde17..65caf968 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -1,5 +1,5 @@ import { setLogLevel } from '../utils' -import { runPlugins, runPluginsBatch, setupPlugins } from '../plugins' +import { runPlugins, runPluginsOnBatch, setupPlugins } from '../plugins' import { createServer } from '../server' import { PluginsServerConfig } from '../types' @@ -30,7 +30,7 @@ export async function createWorker(config: PluginsServerConfig): Promise) } if (task === 'processEvents') { - const processedEvents = await runPluginsBatch(server, args.events) + const processedEvents = await runPluginsOnBatch(server, args.events) // must clone the object, as we may get from VM2 something like { ..., properties: Proxy {} } return cloneObject(processedEvents as any[]) } diff --git a/tests/plugins.test.ts b/tests/plugins.test.ts index 0f58f3cc..c3c0a8cd 100644 --- a/tests/plugins.test.ts +++ b/tests/plugins.test.ts @@ -52,7 +52,7 @@ test('setupPlugins and runPlugins', async () => { }, }) expect(pluginConfig.vm).toBeDefined() - expect(Object.keys(pluginConfig.vm!.methods)).toEqual(['processEvent']) + expect(Object.keys(pluginConfig.vm!.methods)).toEqual(['processEvent', 'processEvents']) expect(setError).toHaveBeenCalled() expect(setError.mock.calls[0][0]).toEqual(mockServer) diff --git a/tests/vm.test.ts b/tests/vm.test.ts index 7a802749..6ee83dcd 100644 --- a/tests/vm.test.ts +++ b/tests/vm.test.ts @@ -1,4 +1,4 @@ -import { createPluginConfigVM, prepareForRun } from '../src/vm' +import { createPluginConfigVM } from '../src/vm' import { PluginConfig, PluginsServer, Plugin } from '../src/types' import { PluginEvent } from 'posthog-plugins' import { createServer } from '../src/server' @@ -492,96 +492,3 @@ test('attachments', async () => { expect(event.properties).toEqual(attachments) }) - -test('prepareForRun without token', async () => { - const indexJs = ` - async function processEvent (event, meta) { - event.properties = { - posthog: posthog - } - return event - } - ` - const pluginConfig = { ...mockConfig } - const vm = createPluginConfigVM(mockServer, pluginConfig, indexJs) - pluginConfig.vm = vm - const event: PluginEvent = { - ...defaultEvent, - event: 'prepareForRun event', - properties: {}, - } - const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) as ( - event: PluginEvent - ) => Promise - - expect(processEvent).toBeDefined() - - await processEvent!(event) - - expect(event.properties!.posthog).toEqual(null) -}) - -test('prepareForRun with token gets posthog', async () => { - const indexJs = ` - async function processEvent (event, meta) { - event.properties = { - posthog: posthog - } - return event - } - ` - const pluginConfig = { ...mockConfig } - const vm = createPluginConfigVM(mockServer, pluginConfig, indexJs) - pluginConfig.vm = vm - const event: PluginEvent = { - ...defaultEvent, - event: 'prepareForRun event', - properties: { - token: 'posthog-token', - }, - } - const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) as ( - event: PluginEvent - ) => Promise - expect(processEvent).toBeDefined() - - await processEvent!(event) - - expect(event.properties!.posthog.capture).toBeDefined() - expect(event.properties!.posthog.identify).toBeDefined() -}) - -test('posthog.capture', async () => { - const indexJs = ` - async function processEvent (event, meta) { - posthog.capture('random-event', { prop: 'value' }) - return event - } - ` - const pluginConfig = { ...mockConfig } - const vm = createPluginConfigVM(mockServer, pluginConfig, indexJs) - pluginConfig.vm = vm - const event: PluginEvent = { - ...defaultEvent, - event: 'prepareForRun event', - properties: { - // needs a token in the event - token: 'posthog-token', - }, - } - const processEvent = prepareForRun(mockServer, pluginConfig.team_id, pluginConfig, 'processEvent', event) as ( - event: PluginEvent - ) => Promise - expect(processEvent).toBeDefined() - await processEvent!(event) - - expect((fetch as any).mock.calls[0][0]).toContain('http://localhost/e/?ip=1&_=') - expect((fetch as any).mock.calls[0][1].body).toContain('data=') - expect((fetch as any).mock.calls[0][1].body).toContain('&compression=lz64') - expect((fetch as any).mock.calls[0][1]).toMatchObject({ - credentials: 'omit', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'POST', - mode: 'no-cors', - }) -}) From 29a89e33848ffbfbf4b6304a4ab66d2d346e8220 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 14:55:51 +0100 Subject: [PATCH 47/61] less round in tests, increase timeout --- tests/worker.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/worker.test.ts b/tests/worker.test.ts index 9b779e00..f4da18ef 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -7,7 +7,7 @@ import * as os from 'os' import { LogLevel } from '../src/types' jest.mock('../src/sql') -jest.setTimeout(300000) // 300 sec timeout +jest.setTimeout(600000) // 600 sec timeout function processOneEvent( processEvent: (event: PluginEvent) => Promise, @@ -77,7 +77,7 @@ test('piscina worker test', async () => { const coreCount = os.cpus().length const workerThreads = [1, 2, 4, 8, 12, 16].filter((threads) => (isDevRun ? threads <= 8 : threads <= coreCount)) - const rounds = isDevRun ? 1 : 3 + const rounds = 1 const tests: { testName: string; events: number; testCode: string }[] = [ { From 9f65f77387c06f71056be2285f81ab96336a64ce Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 15:03:12 +0100 Subject: [PATCH 48/61] refactor --- src/plugins.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index a3d42858..ad723a6d 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -180,8 +180,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr } export async function runPlugins(server: PluginsServer, event: PluginEvent): Promise { - const pluginsToRun = server.pluginConfigsPerTeam.get(event.team_id) || server.defaultConfigs - + const pluginsToRun = getPluginsForTeam(server, event.team_id) let returnedEvent: PluginEvent | null = event for (const pluginConfig of pluginsToRun.reverse()) { @@ -221,7 +220,7 @@ export async function runPluginsOnBatch(server: PluginsServer, events: PluginEve let allReturnedEvents: PluginEvent[] = [] for (const [teamId, teamEvents] of eventsByTeam.entries()) { - const pluginsToRun = server.pluginConfigsPerTeam.get(teamId) || server.defaultConfigs + const pluginsToRun = getPluginsForTeam(server, teamId) let returnedEvents: PluginEvent[] = teamEvents @@ -246,3 +245,7 @@ export async function runPluginsOnBatch(server: PluginsServer, events: PluginEve return allReturnedEvents } + +function getPluginsForTeam(server: PluginsServer, teamId: number): PluginConfig[] { + return server.pluginConfigsPerTeam.get(teamId) || server.defaultConfigs +} From 9f0a7bad737cfaa370d89af7ac7fef5fda4fbbd5 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 15:10:31 +0100 Subject: [PATCH 49/61] slight cleanup --- src/plugins.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index ad723a6d..c95df580 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -185,17 +185,17 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro for (const pluginConfig of pluginsToRun.reverse()) { if (pluginConfig.vm?.methods?.processEvent) { + let errored = false const { processEvent } = pluginConfig.vm.methods const startTime = performance.now() try { returnedEvent = (await processEvent(returnedEvent)) || null - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms) } catch (error) { + errored = true await processError(server, pluginConfig, error, returnedEvent) - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms, true) } + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms, errored) if (!returnedEvent) { return null @@ -228,15 +228,15 @@ export async function runPluginsOnBatch(server: PluginsServer, events: PluginEve if (pluginConfig.vm?.methods?.processEvents && returnedEvents.length > 0) { const { processEvents } = pluginConfig.vm.methods const startTime = performance.now() + let errored = false try { returnedEvents = (await processEvents(returnedEvents)) || [] - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms) } catch (error) { + errored = true await processError(server, pluginConfig, error, returnedEvents[0]) - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms, true) } + const ms = Math.round((performance.now() - startTime) * 1000) / 1000 + logTime(pluginConfig.plugin?.name || 'noname', ms, errored) } } From c4c8ac8921ff5391d5808d0058a62a5b7cb6e457 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 15:25:12 +0100 Subject: [PATCH 50/61] rename `processEvents` to `processEventBatch` --- src/plugins.ts | 10 +++--- src/types.ts | 2 +- src/vm.ts | 14 ++++---- src/worker/worker.ts | 4 +-- tests/plugins.test.ts | 2 +- tests/vm.test.ts | 76 +++++++++++++++++++++---------------------- tests/worker.test.ts | 8 ++--- 7 files changed, 58 insertions(+), 58 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index c95df580..2924dd99 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -206,10 +206,10 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro return returnedEvent } -export async function runPluginsOnBatch(server: PluginsServer, events: PluginEvent[]): Promise { +export async function runPluginsOnBatch(server: PluginsServer, batch: PluginEvent[]): Promise { const eventsByTeam = new Map() - for (const event of events) { + for (const event of batch) { if (eventsByTeam.has(event.team_id)) { eventsByTeam.get(event.team_id)!.push(event) } else { @@ -225,12 +225,12 @@ export async function runPluginsOnBatch(server: PluginsServer, events: PluginEve let returnedEvents: PluginEvent[] = teamEvents for (const pluginConfig of pluginsToRun.reverse()) { - if (pluginConfig.vm?.methods?.processEvents && returnedEvents.length > 0) { - const { processEvents } = pluginConfig.vm.methods + if (pluginConfig.vm?.methods?.processEventBatch && returnedEvents.length > 0) { + const { processEventBatch } = pluginConfig.vm.methods const startTime = performance.now() let errored = false try { - returnedEvents = (await processEvents(returnedEvents)) || [] + returnedEvents = (await processEventBatch(returnedEvents)) || [] } catch (error) { errored = true await processError(server, pluginConfig, error, returnedEvents[0]) diff --git a/src/types.ts b/src/types.ts index 437a71b3..d753cd41 100644 --- a/src/types.ts +++ b/src/types.ts @@ -113,6 +113,6 @@ export interface PluginConfigVMReponse { vm: VM methods: { processEvent: (event: PluginEvent) => Promise - processEvents: (events: PluginEvent[]) => Promise + processEventBatch: (batch: PluginEvent[]) => Promise } } diff --git a/src/vm.ts b/src/vm.ts index f8e2e9ba..4a64b8ca 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -65,25 +65,25 @@ export function createPluginConfigVM( // run the plugin setup script, if present __callWithMeta('setupPlugin'); - // we have processEvent, but not processEvents - if (!__getExported('processEvents') && __getExported('processEvent')) { - exports.processEvents = async function processEvents (events, meta) { + // we have processEvent, but not processEventBatch + if (!__getExported('processEventBatch') && __getExported('processEvent')) { + exports.processEventBatch = async function processEventBatch (events, meta) { const processEvent = __getExported('processEvent'); const pArray = events.map(async event => await processEvent(event, meta)) const response = await Promise.all(pArray); return response.filter(r => r) } - // we have processEvents, but not processEvent - } else if (!__getExported('processEvent') && __getExported('processEvents')) { + // we have processEventBatch, but not processEvent + } else if (!__getExported('processEvent') && __getExported('processEventBatch')) { exports.processEvent = async function processEvent (event, meta) { - return (await (__getExported('processEvents'))([event], meta))?.[0] + return (await (__getExported('processEventBatch'))([event], meta))?.[0] } } // export various functions const __methods = { processEvent: __bindMeta('processEvent'), - processEvents: __bindMeta('processEvents') + processEventBatch: __bindMeta('processEventBatch') }; ` ) diff --git a/src/worker/worker.ts b/src/worker/worker.ts index 65caf968..9d91b5bb 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -29,8 +29,8 @@ export async function createWorker(config: PluginsServerConfig): Promise) } - if (task === 'processEvents') { - const processedEvents = await runPluginsOnBatch(server, args.events) + if (task === 'processEventBatch') { + const processedEvents = await runPluginsOnBatch(server, args.batch) // must clone the object, as we may get from VM2 something like { ..., properties: Proxy {} } return cloneObject(processedEvents as any[]) } diff --git a/tests/plugins.test.ts b/tests/plugins.test.ts index c3c0a8cd..88a55510 100644 --- a/tests/plugins.test.ts +++ b/tests/plugins.test.ts @@ -52,7 +52,7 @@ test('setupPlugins and runPlugins', async () => { }, }) expect(pluginConfig.vm).toBeDefined() - expect(Object.keys(pluginConfig.vm!.methods)).toEqual(['processEvent', 'processEvents']) + expect(Object.keys(pluginConfig.vm!.methods)).toEqual(['processEvent', 'processEventBatch']) expect(setError).toHaveBeenCalled() expect(setError.mock.calls[0][0]).toEqual(mockServer) diff --git a/tests/vm.test.ts b/tests/vm.test.ts index 6ee83dcd..9b505520 100644 --- a/tests/vm.test.ts +++ b/tests/vm.test.ts @@ -55,9 +55,9 @@ test('empty plugins', async () => { const vm = createPluginConfigVM(mockServer, mockConfig, indexJs, libJs) expect(Object.keys(vm).sort()).toEqual(['methods', 'vm']) - expect(Object.keys(vm.methods).sort()).toEqual(['processEvent', 'processEvents']) + expect(Object.keys(vm.methods).sort()).toEqual(['processEvent', 'processEventBatch']) expect(vm.methods.processEvent).toEqual(undefined) - expect(vm.methods.processEvents).toEqual(undefined) + expect(vm.methods.processEventBatch).toEqual(undefined) }) test('processEvent', async () => { @@ -69,7 +69,7 @@ test('processEvent', async () => { ` const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) expect(vm.methods.processEvent).not.toEqual(undefined) - expect(vm.methods.processEvents).not.toEqual(undefined) + expect(vm.methods.processEventBatch).not.toEqual(undefined) const event: PluginEvent = { ...defaultEvent, @@ -80,16 +80,16 @@ test('processEvent', async () => { expect(newEvent.event).toEqual('changed event') expect(newEvent).toBe(event) - const events: PluginEvent[] = [ + const batch: PluginEvent[] = [ { ...defaultEvent, event: 'original event', }, ] - const newEvents = await vm.methods.processEvents(events) - expect(events[0].event).toEqual('changed event') - expect(newEvents[0].event).toEqual('changed event') - expect(newEvents[0]).toBe(events[0]) + const newBatch = await vm.methods.processEventBatch(batch) + expect(batch[0].event).toEqual('changed event') + expect(newBatch[0].event).toEqual('changed event') + expect(newBatch[0]).toBe(batch[0]) }) test('async processEvent', async () => { @@ -101,7 +101,7 @@ test('async processEvent', async () => { ` const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) expect(vm.methods.processEvent).not.toEqual(undefined) - expect(vm.methods.processEvents).not.toEqual(undefined) + expect(vm.methods.processEventBatch).not.toEqual(undefined) const event: PluginEvent = { ...defaultEvent, @@ -112,21 +112,21 @@ test('async processEvent', async () => { expect(newEvent.event).toEqual('changed event') expect(newEvent).toBe(event) - const events: PluginEvent[] = [ + const batch: PluginEvent[] = [ { ...defaultEvent, event: 'original event', }, ] - const newEvents = await vm.methods.processEvents(events) - expect(events[0].event).toEqual('changed event') - expect(newEvents[0].event).toEqual('changed event') - expect(newEvents[0]).toBe(events[0]) + const newBatch = await vm.methods.processEventBatch(batch) + expect(batch[0].event).toEqual('changed event') + expect(newBatch[0].event).toEqual('changed event') + expect(newBatch[0]).toBe(batch[0]) }) -test('processEvents', async () => { +test('processEventBatch', async () => { const indexJs = ` - function processEvents (events, meta) { + function processEventBatch (events, meta) { return events.map(event => { event.event = 'changed event' return event @@ -135,7 +135,7 @@ test('processEvents', async () => { ` const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) expect(vm.methods.processEvent).not.toEqual(undefined) - expect(vm.methods.processEvents).not.toEqual(undefined) + expect(vm.methods.processEventBatch).not.toEqual(undefined) const event: PluginEvent = { ...defaultEvent, @@ -146,21 +146,21 @@ test('processEvents', async () => { expect(newEvent.event).toEqual('changed event') expect(newEvent).toBe(event) - const events: PluginEvent[] = [ + const batch: PluginEvent[] = [ { ...defaultEvent, event: 'original event', }, ] - const newEvents = await vm.methods.processEvents(events) - expect(events[0].event).toEqual('changed event') - expect(newEvents[0].event).toEqual('changed event') - expect(newEvents[0]).toBe(events[0]) + const newBatch = await vm.methods.processEventBatch(batch) + expect(batch[0].event).toEqual('changed event') + expect(newBatch[0].event).toEqual('changed event') + expect(newBatch[0]).toBe(batch[0]) }) -test('async processEvents', async () => { +test('async processEventBatch', async () => { const indexJs = ` - async function processEvents (events, meta) { + async function processEventBatch (events, meta) { return events.map(event => { event.event = 'changed event' return event @@ -169,7 +169,7 @@ test('async processEvents', async () => { ` const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) expect(vm.methods.processEvent).not.toEqual(undefined) - expect(vm.methods.processEvents).not.toEqual(undefined) + expect(vm.methods.processEventBatch).not.toEqual(undefined) const event: PluginEvent = { ...defaultEvent, @@ -180,25 +180,25 @@ test('async processEvents', async () => { expect(newEvent.event).toEqual('changed event') expect(newEvent).toBe(event) - const events: PluginEvent[] = [ + const batch: PluginEvent[] = [ { ...defaultEvent, event: 'original event', }, ] - const newEvents = await vm.methods.processEvents(events) - expect(events[0].event).toEqual('changed event') - expect(newEvents[0].event).toEqual('changed event') - expect(newEvents[0]).toBe(events[0]) + const newBatch = await vm.methods.processEventBatch(batch) + expect(batch[0].event).toEqual('changed event') + expect(newBatch[0].event).toEqual('changed event') + expect(newBatch[0]).toBe(batch[0]) }) -test('processEvent && processEvents', async () => { +test('processEvent && processEventBatch', async () => { const indexJs = ` function processEvent (event, meta) { event.event = 'changed event 1' return event } - function processEvents (events, meta) { + function processEventBatch (events, meta) { return events.map(event => { event.event = 'changed event 2' return event @@ -207,7 +207,7 @@ test('processEvent && processEvents', async () => { ` const vm = createPluginConfigVM(mockServer, mockConfig, indexJs) expect(vm.methods.processEvent).not.toEqual(undefined) - expect(vm.methods.processEvents).not.toEqual(undefined) + expect(vm.methods.processEventBatch).not.toEqual(undefined) const event: PluginEvent = { ...defaultEvent, @@ -218,16 +218,16 @@ test('processEvent && processEvents', async () => { expect(newEvent.event).toEqual('changed event 1') expect(newEvent).toBe(event) - const events: PluginEvent[] = [ + const batch: PluginEvent[] = [ { ...defaultEvent, event: 'original event', }, ] - const newEvents = await vm.methods.processEvents(events) - expect(events[0].event).toEqual('changed event 2') - expect(newEvents[0].event).toEqual('changed event 2') - expect(newEvents[0]).toBe(events[0]) + const newBatch = await vm.methods.processEventBatch(batch) + expect(batch[0].event).toEqual('changed event 2') + expect(newBatch[0].event).toEqual('changed event 2') + expect(newBatch[0]).toBe(batch[0]) }) test('processEvent without returning', async () => { diff --git a/tests/worker.test.ts b/tests/worker.test.ts index f4da18ef..8bd9a320 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -27,7 +27,7 @@ function processOneEvent( } function processOneBatch( - processEvents: (events: PluginEvent[]) => Promise, + processEventBatch: (batch: PluginEvent[]) => Promise, batchSize: number, batchIndex: number ): Promise { @@ -41,21 +41,21 @@ function processOneBatch( properties: { key: 'value', batchIndex, indexInBatch: i }, })) - return processEvents(events) + return processEventBatch(events) } async function processCountEvents(piscina: ReturnType, count: number, batchSize = 1) { const maxPromises = 1000 const promises = Array(maxPromises) const processEvent = (event: PluginEvent) => piscina.runTask({ task: 'processEvent', args: { event } }) - const processEvents = (events: PluginEvent[]) => piscina.runTask({ task: 'processEvents', args: { events } }) + const processEventBatch = (batch: PluginEvent[]) => piscina.runTask({ task: 'processEventBatch', args: { batch } }) const groups = Math.ceil(count / maxPromises) for (let j = 0; j < groups; j++) { const groupCount = groups === 1 ? count : j === groups - 1 ? count % maxPromises : maxPromises for (let i = 0; i < groupCount; i++) { promises[i] = - batchSize === 1 ? processOneEvent(processEvent, i) : processOneBatch(processEvents, batchSize, i) + batchSize === 1 ? processOneEvent(processEvent, i) : processOneBatch(processEventBatch, batchSize, i) } await Promise.all(promises) } From 539a6d38f4b7e0fd8e7f455e46f3c5d024b2fa30 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 8 Dec 2020 15:41:35 +0100 Subject: [PATCH 51/61] only wait in processEventBatch if needed --- src/vm.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/vm.ts b/src/vm.ts index 4a64b8ca..fba4f34c 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -67,10 +67,17 @@ export function createPluginConfigVM( // we have processEvent, but not processEventBatch if (!__getExported('processEventBatch') && __getExported('processEvent')) { - exports.processEventBatch = async function processEventBatch (events, meta) { + exports.processEventBatch = async function processEventBatch (batch, meta) { const processEvent = __getExported('processEvent'); - const pArray = events.map(async event => await processEvent(event, meta)) - const response = await Promise.all(pArray); + let waitFor = false + const processedEvents = batch.map(event => { + const e = processEvent(event, meta) + if (e && typeof e.then !== 'undefined') { + waitFor = true + } + return e + }) + const response = waitFor ? (await Promise.all(processedEvents)) : processedEvents; return response.filter(r => r) } // we have processEventBatch, but not processEvent From 05976babceb68e435ad0a79887ab6c77cc587ce4 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 07:11:20 +0100 Subject: [PATCH 52/61] Order file extensions more readably in lint-staged --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d5cc525d..a65927cf 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,6 @@ } }, "lint-staged": { - "*.{js,css,scss,ts}": "prettier --write" + "*.{js,ts,css,scss}": "prettier --write" } } From 9193c54175a537d2ce170b4ea7c8f99d91d8b11f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 09:15:35 +0100 Subject: [PATCH 53/61] Prettier --- src/worker/worker.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/worker/worker.ts b/src/worker/worker.ts index e44fac22..a80781a7 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -1,4 +1,3 @@ - import { runPlugins, runPluginsOnBatch, setupPlugins } from '../plugins' import { cloneObject, setLogLevel } from '../utils' import { createServer } from '../server' From 015e15f904c4c3faf6aa6ed85527285a648a0a39 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 9 Dec 2020 09:38:50 +0100 Subject: [PATCH 54/61] remove time rounding --- src/plugins.ts | 6 ++---- tests/worker.test.ts | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index 2924dd99..cab91f50 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -194,8 +194,7 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro errored = true await processError(server, pluginConfig, error, returnedEvent) } - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms, errored) + logTime(pluginConfig.plugin?.name || 'noname', startTime - performance.now(), errored) if (!returnedEvent) { return null @@ -235,8 +234,7 @@ export async function runPluginsOnBatch(server: PluginsServer, batch: PluginEven errored = true await processError(server, pluginConfig, error, returnedEvents[0]) } - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - logTime(pluginConfig.plugin?.name || 'noname', ms, errored) + logTime(pluginConfig.plugin?.name || 'noname', startTime - performance.now(), errored) } } diff --git a/tests/worker.test.ts b/tests/worker.test.ts index 8bd9a320..cdb53772 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -135,8 +135,7 @@ test('piscina worker test', async () => { for (let i = 0; i < rounds; i++) { await processCountEvents(piscina, events / batchSize, batchSize) } - const ms = Math.round((performance.now() - startTime) * 1000) / 1000 - const throughput = Math.round(1000 / (ms / events / rounds)) + const throughput = Math.round(1000 / ((startTime - performance.now()) / events / rounds)) result[`${threads} thread${threads === 1 ? '' : 's'}`] = Math.round(throughput) await piscina.destroy() From bdad14c4def57701140d2635729843b7b6479c28 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 9 Dec 2020 09:40:44 +0100 Subject: [PATCH 55/61] less lookups --- src/plugins.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index cab91f50..f52fff00 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -224,8 +224,8 @@ export async function runPluginsOnBatch(server: PluginsServer, batch: PluginEven let returnedEvents: PluginEvent[] = teamEvents for (const pluginConfig of pluginsToRun.reverse()) { - if (pluginConfig.vm?.methods?.processEventBatch && returnedEvents.length > 0) { - const { processEventBatch } = pluginConfig.vm.methods + const { processEventBatch } = pluginConfig.vm?.methods || {} + if (processEventBatch && returnedEvents.length > 0) { const startTime = performance.now() let errored = false try { From 92b8cbcefd44f3f94315ee8f165026c1b6fa0e83 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 9 Dec 2020 09:46:48 +0100 Subject: [PATCH 56/61] describe test devmode --- tests/worker.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/worker.test.ts b/tests/worker.test.ts index cdb53772..d2ebe446 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -73,6 +73,7 @@ function setupPiscina(workers: number, code: string, tasksPerWorker: number) { test('piscina worker test', async () => { // Uncomment this to become a 10x developer and make the test run just as fast! + // Reduces events by 10x and limits threads to max 8 const isDevRun = false const coreCount = os.cpus().length From 0afd70ebb7b61fc834d96d90b1bbf998ce89ab7f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 9 Dec 2020 09:47:47 +0100 Subject: [PATCH 57/61] add "light" --- tests/worker.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/worker.test.ts b/tests/worker.test.ts index d2ebe446..5f31a510 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -74,10 +74,12 @@ function setupPiscina(workers: number, code: string, tasksPerWorker: number) { test('piscina worker test', async () => { // Uncomment this to become a 10x developer and make the test run just as fast! // Reduces events by 10x and limits threads to max 8 - const isDevRun = false + const isLightDevRun = false const coreCount = os.cpus().length - const workerThreads = [1, 2, 4, 8, 12, 16].filter((threads) => (isDevRun ? threads <= 8 : threads <= coreCount)) + const workerThreads = [1, 2, 4, 8, 12, 16].filter((threads) => + isLightDevRun ? threads <= 8 : threads <= coreCount + ) const rounds = 1 const tests: { testName: string; events: number; testCode: string }[] = [ @@ -117,7 +119,7 @@ test('piscina worker test', async () => { const results: Array> = [] for (const { testName, events: _events, testCode } of tests) { - const events = isDevRun ? _events / 10 : _events + const events = isLightDevRun ? _events / 10 : _events for (const batchSize of [1, 10, 100].filter((size) => size <= events)) { const result: Record = { testName, From 9dc93ac0058f3180ddb376a6dd2a4f0f4e6126eb Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 10:14:05 +0100 Subject: [PATCH 58/61] Remove redundant Math.round --- src/stats.ts | 2 +- tests/worker.test.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 33e6d456..e955d981 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -3,7 +3,7 @@ const histories = new Map>() const historyIndex = new Map() export function logTime(name: string, time: number, error?: boolean): void { - const ms = Math.round(time * 1000) / 1000 + // const ms = Math.round(time * 1000) / 1000 // TODO: add this back with better dev logging. Disabling since this trashes performance tests. // console.log(`Running plugin ${name}: ${error ? 'ERROR IN ' : ''}${ms}ms`) if (!histories.has(name)) { diff --git a/tests/worker.test.ts b/tests/worker.test.ts index 5f31a510..cabbef6d 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -73,7 +73,7 @@ function setupPiscina(workers: number, code: string, tasksPerWorker: number) { test('piscina worker test', async () => { // Uncomment this to become a 10x developer and make the test run just as fast! - // Reduces events by 10x and limits threads to max 8 + // Reduces events by 10x and limits threads to max 8 for quicker development const isLightDevRun = false const coreCount = os.cpus().length @@ -138,8 +138,7 @@ test('piscina worker test', async () => { for (let i = 0; i < rounds; i++) { await processCountEvents(piscina, events / batchSize, batchSize) } - const throughput = Math.round(1000 / ((startTime - performance.now()) / events / rounds)) - result[`${threads} thread${threads === 1 ? '' : 's'}`] = Math.round(throughput) + result[`${threads} thread${threads === 1 ? '' : 's'}`] = Math.round(startTime - performance.now()) await piscina.destroy() } From 0aaee9def5ec4f9def691476ac13c08a8dc9e6f8 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 10:14:20 +0100 Subject: [PATCH 59/61] Run prettier --- .devcontainer/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 8b78f8c9..5b43dcee 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -24,6 +24,7 @@ services: # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db + # Uncomment the next line to use a non-root user for all processes. # user: node # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. @@ -39,6 +40,7 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward MongoDB locally. # (Adding the "ports" property to this file will not forward from a Codespace.) redis: From b43bce72d42d201520411e9d6c569f70ed810fa3 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 10:25:11 +0100 Subject: [PATCH 60/61] Fix performance measurement --- src/plugins.ts | 4 ++-- tests/worker.test.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/plugins.ts b/src/plugins.ts index f52fff00..7231fcc3 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -194,7 +194,7 @@ export async function runPlugins(server: PluginsServer, event: PluginEvent): Pro errored = true await processError(server, pluginConfig, error, returnedEvent) } - logTime(pluginConfig.plugin?.name || 'noname', startTime - performance.now(), errored) + logTime(pluginConfig.plugin?.name || 'noname', performance.now() - startTime, errored) if (!returnedEvent) { return null @@ -234,7 +234,7 @@ export async function runPluginsOnBatch(server: PluginsServer, batch: PluginEven errored = true await processError(server, pluginConfig, error, returnedEvents[0]) } - logTime(pluginConfig.plugin?.name || 'noname', startTime - performance.now(), errored) + logTime(pluginConfig.plugin?.name || 'noname', performance.now() - startTime, errored) } } diff --git a/tests/worker.test.ts b/tests/worker.test.ts index cabbef6d..7d0adcc0 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -138,7 +138,9 @@ test('piscina worker test', async () => { for (let i = 0; i < rounds; i++) { await processCountEvents(piscina, events / batchSize, batchSize) } - result[`${threads} thread${threads === 1 ? '' : 's'}`] = Math.round(startTime - performance.now()) + result[`${threads} thread${threads === 1 ? '' : 's'}`] = Math.round( + 1000 / ((startTime - performance.now()) / events / rounds) + ) await piscina.destroy() } From fe28b450a13b04145936526dabbf0a12a64a7032 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 10:25:41 +0100 Subject: [PATCH 61/61] Prettier but correctly --- .devcontainer/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 5b43dcee..8b78f8c9 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -24,7 +24,6 @@ services: # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db - # Uncomment the next line to use a non-root user for all processes. # user: node # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. @@ -40,7 +39,6 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres - # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward MongoDB locally. # (Adding the "ports" property to this file will not forward from a Codespace.) redis: