From 254392ab93142e3a8d23ef0a65bc01cbc8dc4b17 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 10:33:49 +0100 Subject: [PATCH 001/218] Integrate fastify-postgres and fastify-kafka --- package.json | 2 ++ src/web/server.ts | 26 ++++++++++++++++++++++++-- yarn.lock | 39 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index fe88f89b..5bdba69b 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,8 @@ "dependencies": { "adm-zip": "^0.4.16", "fastify": "^3.8.0", + "fastify-kafka": "^0.3.0", + "fastify-postgres": "^3.3.0", "gunzip-maybe": "^1.4.2", "ioredis": "^4.19.2", "node-fetch": "^2.6.1", diff --git a/src/web/server.ts b/src/web/server.ts index 1359310c..bf777314 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -1,7 +1,29 @@ -import { fastify, FastifyInstance } from 'fastify' +import { fastify, FastifyInstance, FastifyPluginAsync, FastifyPluginCallback } from 'fastify' +import { fastifyPostgres } from 'fastify-postgres' +import fastifyKafka from 'fastify-kafka' -export function buildFastifyInstance(): FastifyInstance { +export interface FastifySettings { + DATABASE_URL: string // Postgres database URL + KAFKA_HOSTS: string +} + +export function buildFastifyInstance({ DATABASE_URL, KAFKA_HOSTS }: FastifySettings): FastifyInstance { const fastifyInstance = fastify() + fastifyInstance.register(fastifyPostgres, { + connectionString: DATABASE_URL, + }) + fastifyInstance.register(fastifyKafka as FastifyPluginAsync, { + producer: { + dr_cb: true, + 'metadata.broker.list': KAFKA_HOSTS, + }, + consumer: { + 'metadata.broker.list': KAFKA_HOSTS, + }, + consumerTopicConf: { + 'auto.offset.reset': 'earliest', + }, + }) return fastifyInstance } diff --git a/yarn.lock b/yarn.lock index 1299485f..6183f496 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1890,7 +1890,7 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== -bindings@^1.5.0: +bindings@^1.3.1, bindings@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== @@ -3052,6 +3052,33 @@ fastify-error@^0.2.0: resolved "https://registry.yarnpkg.com/fastify-error/-/fastify-error-0.2.0.tgz#9a1c28d4f42b6259e7a549671c8e5e2d85660634" integrity sha512-zabxsBatj59ROG0fhP36zNdc5Q1/eYeH9oSF9uvfrurZf8/JKfrJbMcIGrLpLWcf89rS6L91RHWm20A/X85hcA== +fastify-kafka@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/fastify-kafka/-/fastify-kafka-0.3.0.tgz#3fc83b7d0a66370c5d80704acb65bbdb4a9c0644" + integrity sha512-HtzM89XrW6JCQQMWYbzQHgKOr5bLFxqdAAA2IbJ6Q8bueLhaWuTjvfHgWzhzlCOy3rbSQNo+bi8GGpz2KZTDEw== + dependencies: + fastify-plugin "^2.0.0" + node-rdkafka "^2.8.1" + +fastify-plugin@^2.0.0: + version "2.3.4" + resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-2.3.4.tgz#b17abdc36a97877d88101fb86ad8a07f2c07de87" + integrity sha512-I+Oaj6p9oiRozbam30sh39BiuiqBda7yK2nmSPVwDCfIBlKnT8YB3MY+pRQc2Fcd07bf6KPGklHJaQ2Qu81TYQ== + dependencies: + semver "^7.3.2" + +fastify-plugin@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-3.0.0.tgz#cf1b8c8098e3b5a7c8c30e6aeb06903370c054ca" + integrity sha512-ZdCvKEEd92DNLps5n0v231Bha8bkz1DjnPP/aEz37rz/q42Z5JVLmgnqR4DYuNn3NXAO3IDCPyRvgvxtJ4Ym4w== + +fastify-postgres@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/fastify-postgres/-/fastify-postgres-3.3.0.tgz#e23e86957ac826757999b92db98120e9edaf796c" + integrity sha512-B0yoveN1Oo9oLM4phFl5yscmj1Lpq1lD9YXXUIDRAsd0eeJdvphT+bEHARHFnoSQuFr5QGViqtpPIrRDZQxMIA== + dependencies: + fastify-plugin "^3.0.0" + fastify-warning@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/fastify-warning/-/fastify-warning-0.2.0.tgz#e717776026a4493dc9a2befa44db6d17f618008f" @@ -4880,7 +4907,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -nan@^2.12.1: +nan@^2.12.1, nan@^2.14.0: version "2.14.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== @@ -4939,6 +4966,14 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" +node-rdkafka@^2.8.1: + version "2.9.1" + resolved "https://registry.yarnpkg.com/node-rdkafka/-/node-rdkafka-2.9.1.tgz#ac711bc7845ccb1de87391aec7f27f683d265149" + integrity sha512-C5EVDZlDG+5D8KXiz2zKwEiLWIGW5Z1mkVFRzp13T4mrbXz+ESyjrDSLIj7aoUIi5+T10H9p1wwLZJBh9ivjLg== + dependencies: + bindings "^1.3.1" + nan "^2.14.0" + node-releases@^1.1.65: version "1.1.65" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.65.tgz#52d9579176bd60f23eba05c4438583f341944b81" From 7cd0ff5197fcdf7cac43f91967b4cc77d46e9266 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 16:33:27 +0100 Subject: [PATCH 002/218] Explicitly add node-rdkafka for its types --- package.json | 1 + yarn.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 5bdba69b..e2affe04 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "gunzip-maybe": "^1.4.2", "ioredis": "^4.19.2", "node-fetch": "^2.6.1", + "node-rdkafka": "^2.9.1", "node-schedule": "^1.3.2", "pg": "^8.4.2", "posthog-js-lite": "^0.0.5", diff --git a/yarn.lock b/yarn.lock index 6183f496..2a7e2cea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4966,7 +4966,7 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" -node-rdkafka@^2.8.1: +node-rdkafka@^2.8.1, node-rdkafka@^2.9.1: version "2.9.1" resolved "https://registry.yarnpkg.com/node-rdkafka/-/node-rdkafka-2.9.1.tgz#ac711bc7845ccb1de87391aec7f27f683d265149" integrity sha512-C5EVDZlDG+5D8KXiz2zKwEiLWIGW5Z1mkVFRzp13T4mrbXz+ESyjrDSLIj7aoUIi5+T10H9p1wwLZJBh9ivjLg== From 0b87dfbf0a2cce52d96fa4cca9b94f1cb4373ea3 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 16:33:32 +0100 Subject: [PATCH 003/218] Port UUIDT from Python --- src/web/utils.ts | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/web/utils.ts diff --git a/src/web/utils.ts b/src/web/utils.ts new file mode 100644 index 00000000..6325d930 --- /dev/null +++ b/src/web/utils.ts @@ -0,0 +1,75 @@ +import { stringify as uuidStringify } from 'uuid' + +/** + * UUID (mostly) sortable by generation time. + * + * This doesn't adhere to any official UUID version spec, but it is superior as a primary key: + * to incremented integers (as they can reveal sensitive business information about usage volumes and patterns), + * to UUID v4 (as the complete randomness of v4 makes its indexing performance suboptimal), + * and to UUID v1 (as despite being time-based it can't be used practically for sorting by generation time). + * + * Order can be messed up if system clock is changed or if more than 65ย 536 IDs are generated per millisecond + * (that's over 5 trillion events per day), but it should be largely safe to assume that these are time-sortable. + * + * Anatomy: + * - 6 bytes - Unix time milliseconds unsigned integer + * - 2 bytes - autoincremented series unsigned integer (per millisecond, rolls over to 0 after reaching 65 535 UUIDs in one ms) + * - 8 bytes - securely random gibberish + * + * Loosely based on [Segment's KSUID](https://github.com/segmentio/ksuid) and + * on [Twitter's snowflake ID](https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html). + * Ported from the PostHog Django app. + */ +export class UUIDT { + static currentSeriesPerMs: Map = new Map() + static crypto = new Crypto() + + /** Get per-millisecond series integer in range [0-65536). */ + static getSeries(unixTimeMs: number): number { + const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) + if (UUIDT.currentSeriesPerMs.size > 10_000) { + // Clear class dict periodically + UUIDT.currentSeriesPerMs.clear() + } + const nextSeries = typeof series === 'number' ? (series + 1) % 65_536 : 0 + UUIDT.currentSeriesPerMs.set(unixTimeMs, nextSeries) + return nextSeries + } + + array: Uint8Array + + constructor(unixTimeMs?: number) { + if (!unixTimeMs) { + unixTimeMs = Date.now() + } + + this.array = new Uint8Array(16) + // 48 bits for time, WILL FAIL in 10 895 CE + let shiftedUnixTimeMs = unixTimeMs + for (let i = 5; i >= 0; i--) { + this.array[i] = shiftedUnixTimeMs & 0b11111111 // use last 8 binary digits + shiftedUnixTimeMs >>= 8 // remove these last 8 binary digits + } + // 16 bits for series + let series = UUIDT.getSeries(unixTimeMs) + for (let i = 7; i >= 6; i--) { + this.array[i] = series & 0b11111111 // use last 8 binary digits + series >>= 8 // remove these last 8 binary digits + } + // 64 bits for random gibberish + this.array.set(UUIDT.crypto.getRandomValues(new Uint32Array(2)), 8) + } + + toNumber(): number { + let value = 0 + for (const byte of this.array) { + value <<= 8 + value += byte + } + return value + } + + toString(): string { + return uuidStringify(this.array) + } +} From bb3f40a79d53b7ab6ede67b02d0ea58c2aaa067b Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 16:33:57 +0100 Subject: [PATCH 004/218] Consume events from Kafka --- src/types.ts | 4 ++- src/web/process-event.ts | 60 ++++++++++++++++++++++++++++++++++++++++ src/web/server.ts | 39 ++++++++++++++------------ src/web/topics.ts | 5 ++++ 4 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 src/web/process-event.ts create mode 100644 src/web/topics.ts diff --git a/src/types.ts b/src/types.ts index d5602f1b..1b0d81d6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,7 +5,9 @@ import { VM, VMScript } from 'vm2' export interface PluginsServerConfig { CELERY_DEFAULT_QUEUE: string - DATABASE_URL: string + DATABASE_URL: string // Postgres database URL + KAFKA_HOSTS: string // comma-delimited Kafka hosts + EE_ENABLED: boolean // whether EE features, such as Kafka + ClickHouse, are enabled PLUGINS_CELERY_QUEUE: string REDIS_URL: string BASE_DIR: string diff --git a/src/web/process-event.ts b/src/web/process-event.ts new file mode 100644 index 00000000..9b394972 --- /dev/null +++ b/src/web/process-event.ts @@ -0,0 +1,60 @@ +import { Message } from 'node-rdkafka' +import { UUIDT } from './utils' + +export interface PostHogEvent { + distinct_id: string + ip: string + site_url: string + data: Record + team_id: number + now: string + sent_at: string +} + +export function processEventFromKafka(message: Message, commit: () => void): void { + // TODO: time with statsd + const event = JSON.parse(message.value!.toString()) as PostHogEvent + processEventEE( + event.distinct_id, + event.ip, + event.site_url, + event.data, + event.team_id, + new Date(event.now), + event.sent_at ? new Date(event.sent_at) : null + ) + commit() +} + +function processEventEE( + distinct_id: string, + ip: string, + site_url: string, + data: Record, + team_id: number, + now: Date, + sent_at: Date | null +): void { + const properties: Record = data.properties ?? {} + if (data['$set']) { + properties['$set'] = data['$set'] + } + + const person_uuid = new UUIDT() + const event_uuid = new UUIDT() + const ts = handle_timestamp(data, now, sent_at) + handle_identify_or_alias(data['event'], properties, distinct_id, team_id) + + if (data['event'] === '$snapshot') { + create_session_recording_event( + event_uuid, + team_id, + distinct_id, + properties['$session_id'], + ts, + properties['$snapshot_data'] + ) + } else { + _capture_ee(event_uuid, person_uuid, ip, site_url, team_id, data['event'], distinct_id, properties, ts) + } +} diff --git a/src/web/server.ts b/src/web/server.ts index bf777314..6e800e68 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -1,29 +1,32 @@ import { fastify, FastifyInstance, FastifyPluginAsync, FastifyPluginCallback } from 'fastify' import { fastifyPostgres } from 'fastify-postgres' import fastifyKafka from 'fastify-kafka' +import { PluginsServerConfig } from 'types' +import { KAFKA_EVENTS_WAL } from './topics' +import { EventEmitter } from 'events' +import { processEventFromKafka } from './process-event' -export interface FastifySettings { - DATABASE_URL: string // Postgres database URL - KAFKA_HOSTS: string -} - -export function buildFastifyInstance({ DATABASE_URL, KAFKA_HOSTS }: FastifySettings): FastifyInstance { +export function buildFastifyInstance({ DATABASE_URL, EE_ENABLED, KAFKA_HOSTS }: PluginsServerConfig): FastifyInstance { const fastifyInstance = fastify() fastifyInstance.register(fastifyPostgres, { connectionString: DATABASE_URL, }) - fastifyInstance.register(fastifyKafka as FastifyPluginAsync, { - producer: { - dr_cb: true, - 'metadata.broker.list': KAFKA_HOSTS, - }, - consumer: { - 'metadata.broker.list': KAFKA_HOSTS, - }, - consumerTopicConf: { - 'auto.offset.reset': 'earliest', - }, - }) + if (EE_ENABLED) { + fastifyInstance.register(fastifyKafka as FastifyPluginAsync, { + producer: { + dr_cb: true, + 'metadata.broker.list': KAFKA_HOSTS, + }, + consumer: { + 'metadata.broker.list': KAFKA_HOSTS, + }, + consumerTopicConf: { + 'auto.offset.reset': 'earliest', + }, + }) + fastifyInstance.kafka.subscribe([KAFKA_EVENTS_WAL]) + ;((fastifyInstance.kafka as unknown) as EventEmitter).on(KAFKA_EVENTS_WAL, processEventFromKafka) + } return fastifyInstance } diff --git a/src/web/topics.ts b/src/web/topics.ts new file mode 100644 index 00000000..c90ebf55 --- /dev/null +++ b/src/web/topics.ts @@ -0,0 +1,5 @@ +export const KAFKA_EVENTS = 'clickhouse_events_proto' +export const KAFKA_PERSON = 'clickhouse_person' +export const KAFKA_PERSON_UNIQUE_ID = 'clickhouse_person_unique_id' +export const KAFKA_SESSION_RECORDING_EVENTS = 'clickhouse_session_recording_events' +export const KAFKA_EVENTS_WAL = 'events_write_ahead_log' From 3a1412c4a9d2c3a25be154e620187889faf3dc67 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 16:39:59 +0100 Subject: [PATCH 005/218] Update yarn.lock --- yarn.lock | 7 ------- 1 file changed, 7 deletions(-) diff --git a/yarn.lock b/yarn.lock index bcd91add..d0c358dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1275,13 +1275,6 @@ dependencies: "@types/node" "*" -"@types/gunzip-maybe@^1.4.0": - version "1.4.0" - resolved "https://registry.yarnpkg.com/@types/gunzip-maybe/-/gunzip-maybe-1.4.0.tgz#9410fd15ff68eca8907b7b9198e63e2a7c14d511" - integrity sha512-dFP9GrYAR9KhsjTkWJ8q8Gsfql75YIKcg9DuQOj/IrlPzR7W+1zX+cclw1McV82UXAQ+Lpufvgk3e9bC8+HzgA== - dependencies: - "@types/node" "*" - "@types/ioredis@^4.17.7": version "4.17.7" resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.17.7.tgz#5007a56380fd387315efabc6c777c050feabb121" From be869206784ba6eddfb236f31d0b5af8863f6fa4 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 16:59:00 +0100 Subject: [PATCH 006/218] Port handle_timestamp --- src/web/process-event.ts | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/web/process-event.ts b/src/web/process-event.ts index 9b394972..9b775f52 100644 --- a/src/web/process-event.ts +++ b/src/web/process-event.ts @@ -1,11 +1,13 @@ import { Message } from 'node-rdkafka' import { UUIDT } from './utils' +export type Data = Record + export interface PostHogEvent { distinct_id: string ip: string site_url: string - data: Record + data: Data team_id: number now: string sent_at: string @@ -30,7 +32,7 @@ function processEventEE( distinct_id: string, ip: string, site_url: string, - data: Record, + data: Data, team_id: number, now: Date, sent_at: Date | null @@ -58,3 +60,24 @@ function processEventEE( _capture_ee(event_uuid, person_uuid, ip, site_url, team_id, data['event'], distinct_id, properties, ts) } } + +function handle_timestamp(data: Data, now: Date, sent_at: Date | null): Date { + if (data["timestamp"]) { + if (sent_at) { + // sent_at - timestamp == now - x + // x = now + (timestamp - sent_at) + try { + // timestamp and sent_at must both be in the same format: either both with or both without timezones + // otherwise we can't get a diff to add to now + return now + (new Date(data["timestamp"]) - sent_at) + } catch (e) { + capture_exception(e) + } + } + return new Date(data["timestamp"]) + } + if (!data["offset"]) { + return now - relativedelta(microseconds=data["offset"] * 1000) + } + return now +} From 43af8a083ab0844e716266c23b0b332e8a1891e0 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 17:50:33 +0100 Subject: [PATCH 007/218] Enhance startFastifyInstance with config --- src/server.ts | 2 +- src/types.ts | 4 ++-- src/web/server.ts | 10 +++------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/server.ts b/src/server.ts index acfac4ea..05d39cbf 100644 --- a/src/server.ts +++ b/src/server.ts @@ -72,7 +72,7 @@ export async function startPluginsServer(config: PluginsServerConfig): Promise { +export async function startFastifyInstance(config: PluginsServerConfig): Promise { console.info(`๐Ÿ‘พ Starting web serverโ€ฆ`) - const fastifyInstance = buildFastifyInstance() + const fastifyInstance = buildFastifyInstance(config) try { - const address = await fastifyInstance.listen(port, hostname) + const address = await fastifyInstance.listen(config.WEB_PORT ?? 3008, config.WEB_HOSTNAME) console.info(`โœ… Web server listening on ${address}!`) } catch (e) { console.error(`๐Ÿ›‘ Web server could not start! ${e}`) From d2aaa2e3500e625cb9565b41df5dbcf8fcb4e5ec Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 2 Dec 2020 18:23:55 +0100 Subject: [PATCH 008/218] Prettier --- src/web/process-event.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/web/process-event.ts b/src/web/process-event.ts index 9b775f52..78b8f206 100644 --- a/src/web/process-event.ts +++ b/src/web/process-event.ts @@ -62,22 +62,22 @@ function processEventEE( } function handle_timestamp(data: Data, now: Date, sent_at: Date | null): Date { - if (data["timestamp"]) { + if (data['timestamp']) { if (sent_at) { // sent_at - timestamp == now - x // x = now + (timestamp - sent_at) try { // timestamp and sent_at must both be in the same format: either both with or both without timezones // otherwise we can't get a diff to add to now - return now + (new Date(data["timestamp"]) - sent_at) + return now + (new Date(data['timestamp']) - sent_at) } catch (e) { capture_exception(e) } } - return new Date(data["timestamp"]) + return new Date(data['timestamp']) } - if (!data["offset"]) { - return now - relativedelta(microseconds=data["offset"] * 1000) + if (!data['offset']) { + return now - relativedelta((microseconds = data['offset'] * 1000)) } return now } From 7a1500522adad75ed87b3cdf6c45a167d8844e88 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 3 Dec 2020 11:47:56 +0100 Subject: [PATCH 009/218] Handle dates with Luxon for similarity to Python's datetime --- package.json | 2 ++ src/types.ts | 2 +- src/web/process-event.ts | 25 +++++++++++++------------ yarn.lock | 10 ++++++++++ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 3fcef759..3ce37873 100644 --- a/package.json +++ b/package.json @@ -25,11 +25,13 @@ "repository": "https://github.com/PostHog/posthog-plugin-server", "license": "MIT", "dependencies": { + "@types/luxon": "^1.25.0", "adm-zip": "^0.4.16", "fastify": "^3.8.0", "fastify-kafka": "^0.3.0", "fastify-postgres": "^3.3.0", "ioredis": "^4.19.2", + "luxon": "^1.25.0", "node-fetch": "^2.6.1", "node-rdkafka": "^2.9.1", "node-schedule": "^1.3.2", diff --git a/src/types.ts b/src/types.ts index 7460283b..65a48b2e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,7 +7,7 @@ export interface PluginsServerConfig { CELERY_DEFAULT_QUEUE: string DATABASE_URL: string // Postgres database URL KAFKA_HOSTS: string // comma-delimited Kafka hosts - EE_ENABLED: boolean // whether EE features, such as Kafka + ClickHouse, are enabled + EE_ENABLED: boolean // whether EE features Kafka + ClickHouse are enabled PLUGINS_CELERY_QUEUE: string REDIS_URL: string BASE_DIR: string diff --git a/src/web/process-event.ts b/src/web/process-event.ts index 78b8f206..e2e9ba72 100644 --- a/src/web/process-event.ts +++ b/src/web/process-event.ts @@ -1,4 +1,5 @@ import { Message } from 'node-rdkafka' +import { DateTime, Duration } from 'luxon' import { UUIDT } from './utils' export type Data = Record @@ -22,8 +23,8 @@ export function processEventFromKafka(message: Message, commit: () => void): voi event.site_url, event.data, event.team_id, - new Date(event.now), - event.sent_at ? new Date(event.sent_at) : null + DateTime.fromISO(event.now), + event.sent_at ? DateTime.fromISO(event.sent_at) : null ) commit() } @@ -34,8 +35,8 @@ function processEventEE( site_url: string, data: Data, team_id: number, - now: Date, - sent_at: Date | null + now: DateTime, + sent_at: DateTime | null ): void { const properties: Record = data.properties ?? {} if (data['$set']) { @@ -61,23 +62,23 @@ function processEventEE( } } -function handle_timestamp(data: Data, now: Date, sent_at: Date | null): Date { - if (data['timestamp']) { +function handle_timestamp(data: Data, now: DateTime, sent_at: DateTime | null): DateTime { + if (data["timestamp"]) { if (sent_at) { // sent_at - timestamp == now - x // x = now + (timestamp - sent_at) try { // timestamp and sent_at must both be in the same format: either both with or both without timezones // otherwise we can't get a diff to add to now - return now + (new Date(data['timestamp']) - sent_at) - } catch (e) { - capture_exception(e) + return now.plus(DateTime.fromISO(data["timestamp"]).diff(sent_at)) + } catch (error) { + console.error(error) } } - return new Date(data['timestamp']) + return DateTime.fromISO(data["timestamp"]) } - if (!data['offset']) { - return now - relativedelta((microseconds = data['offset'] * 1000)) + if (!data["offset"]) { + return now.minus(Duration.fromMillis(data["offset"])) } return now } diff --git a/yarn.lock b/yarn.lock index d0c358dd..068906b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1319,6 +1319,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/luxon@^1.25.0": + version "1.25.0" + resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-1.25.0.tgz#3d6fe591fac874f48dd225cb5660b2b785a21a05" + integrity sha512-iIJp2CP6C32gVqI08HIYnzqj55tlLnodIBMCcMf28q9ckqMfMzocCmIzd9JWI/ALLPMUiTkCu1JGv3FFtu6t3g== + "@types/node-fetch@^2.5.7": version "2.5.7" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c" @@ -4789,6 +4794,11 @@ loud-rejection@^1.0.0: currently-unhandled "^0.4.1" signal-exit "^3.0.0" +luxon@^1.25.0: + version "1.25.0" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-1.25.0.tgz#d86219e90bc0102c0eb299d65b2f5e95efe1fe72" + integrity sha512-hEgLurSH8kQRjY6i4YLey+mcKVAWXbDNlZRmM6AgWDJ1cY3atl8Ztf5wEY7VBReFbmGnwQPz7KYJblL8B2k0jQ== + magic-string@^0.25.2: version "0.25.7" resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051" From 73d42d783615ba380c832a5828f6f427edaf65cc Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 3 Dec 2020 11:48:33 +0100 Subject: [PATCH 010/218] Fix UUIDT stringification --- src/__tests__/web/utils.test.ts | 12 +++++++ src/web/utils.ts | 60 +++++++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 src/__tests__/web/utils.test.ts diff --git a/src/__tests__/web/utils.test.ts b/src/__tests__/web/utils.test.ts new file mode 100644 index 00000000..49992fe0 --- /dev/null +++ b/src/__tests__/web/utils.test.ts @@ -0,0 +1,12 @@ +import { UUIDT } from '../../web/utils' + +test('UUIDT', async () => { + const uuidt = new UUIDT() + const uuidtString = uuidt.toString() + // 32 hexadecimal digits + 4 hyphens + expect(uuidtString).toHaveLength(36) + // UTC timestamp matching (roughly) + expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8)) + // series matching + expect(uuidtString.slice(14, 18)).toEqual('0000') +}) diff --git a/src/web/utils.ts b/src/web/utils.ts index 6325d930..848e3da4 100644 --- a/src/web/utils.ts +++ b/src/web/utils.ts @@ -1,4 +1,42 @@ -import { stringify as uuidStringify } from 'uuid' +import { randomBytes } from 'crypto' +import { DateTime } from 'luxon'; + +const byteToHex: string[] = [] +for (let i = 0; i < 256; i++) { + byteToHex.push((i + 0x100).toString(16).substr(1)); +} + +/** + * Convert array of 16 byte values to UUID string format of the form: + * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + * Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js + */ +function uuidStringify(arr: Uint8Array) { + // Note: Be careful editing this code! It's been tuned for performance and works in ways you may not expect. + // See https://github.com/uuidjs/uuid/pull/434 + return ( + byteToHex[arr[0]] + + byteToHex[arr[1]] + + byteToHex[arr[2]] + + byteToHex[arr[3]] + + '-' + + byteToHex[arr[4]] + + byteToHex[arr[5]] + + '-' + + byteToHex[arr[6]] + + byteToHex[arr[7]] + + '-' + + byteToHex[arr[8]] + + byteToHex[arr[9]] + + '-' + + byteToHex[arr[10]] + + byteToHex[arr[11]] + + byteToHex[arr[12]] + + byteToHex[arr[13]] + + byteToHex[arr[14]] + + byteToHex[arr[15]] + ).toLowerCase() +} /** * UUID (mostly) sortable by generation time. @@ -22,7 +60,6 @@ import { stringify as uuidStringify } from 'uuid' */ export class UUIDT { static currentSeriesPerMs: Map = new Map() - static crypto = new Crypto() /** Get per-millisecond series integer in range [0-65536). */ static getSeries(unixTimeMs: number): number { @@ -40,24 +77,27 @@ export class UUIDT { constructor(unixTimeMs?: number) { if (!unixTimeMs) { - unixTimeMs = Date.now() + unixTimeMs = DateTime.utc().toMillis() } - + let series = UUIDT.getSeries(unixTimeMs) + // 64 bits (8 bytes) total this.array = new Uint8Array(16) // 48 bits for time, WILL FAIL in 10 895 CE - let shiftedUnixTimeMs = unixTimeMs + // XXXXXXXX-XXXX-****-****-************ + // TODO: why the hell are the two leftmost octets (four hexadecimal chars) always zero? for (let i = 5; i >= 0; i--) { - this.array[i] = shiftedUnixTimeMs & 0b11111111 // use last 8 binary digits - shiftedUnixTimeMs >>= 8 // remove these last 8 binary digits + this.array[i] = unixTimeMs & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits + unixTimeMs >>= 8 // remove these last 8 binary digits } // 16 bits for series - let series = UUIDT.getSeries(unixTimeMs) + // ********-****-XXXX-****-************ for (let i = 7; i >= 6; i--) { - this.array[i] = series & 0b11111111 // use last 8 binary digits + this.array[i] = series & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits series >>= 8 // remove these last 8 binary digits } // 64 bits for random gibberish - this.array.set(UUIDT.crypto.getRandomValues(new Uint32Array(2)), 8) + // ********-****-****-XXXX-XXXXXXXXXXXX + this.array.set(randomBytes(8), 8) } toNumber(): number { From 7f02e332406be2282c1aa17e764ea770b224db6e Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 3 Dec 2020 14:44:56 +0100 Subject: [PATCH 011/218] Port _capture_ee --- src/web/process-event.ts | 165 +++++++++++++++++++++++++++++++++++++-- src/web/server.ts | 3 +- src/web/utils.ts | 5 ++ 3 files changed, 164 insertions(+), 9 deletions(-) diff --git a/src/web/process-event.ts b/src/web/process-event.ts index e2e9ba72..a4a47a8c 100644 --- a/src/web/process-event.ts +++ b/src/web/process-event.ts @@ -1,8 +1,9 @@ import { Message } from 'node-rdkafka' import { DateTime, Duration } from 'luxon' -import { UUIDT } from './utils' +import { isLooselyFalsy, UUIDT } from './utils' export type Data = Record +export type Properties = Record export interface PostHogEvent { distinct_id: string @@ -38,7 +39,7 @@ function processEventEE( now: DateTime, sent_at: DateTime | null ): void { - const properties: Record = data.properties ?? {} + const properties: Properties = data.properties ?? {} if (data['$set']) { properties['$set'] = data['$set'] } @@ -46,9 +47,10 @@ function processEventEE( const person_uuid = new UUIDT() const event_uuid = new UUIDT() const ts = handle_timestamp(data, now, sent_at) - handle_identify_or_alias(data['event'], properties, distinct_id, team_id) + // handle_identify_or_alias(data['event'], properties, distinct_id, team_id) if (data['event'] === '$snapshot') { + /* create_session_recording_event( event_uuid, team_id, @@ -57,28 +59,175 @@ function processEventEE( ts, properties['$snapshot_data'] ) + */ } else { _capture_ee(event_uuid, person_uuid, ip, site_url, team_id, data['event'], distinct_id, properties, ts) } } function handle_timestamp(data: Data, now: DateTime, sent_at: DateTime | null): DateTime { - if (data["timestamp"]) { + if (data['timestamp']) { if (sent_at) { // sent_at - timestamp == now - x // x = now + (timestamp - sent_at) try { // timestamp and sent_at must both be in the same format: either both with or both without timezones // otherwise we can't get a diff to add to now - return now.plus(DateTime.fromISO(data["timestamp"]).diff(sent_at)) + return now.plus(DateTime.fromISO(data['timestamp']).diff(sent_at)) } catch (error) { console.error(error) } } - return DateTime.fromISO(data["timestamp"]) + return DateTime.fromISO(data['timestamp']) } - if (!data["offset"]) { - return now.minus(Duration.fromMillis(data["offset"])) + if (!data['offset']) { + return now.minus(Duration.fromMillis(data['offset'])) } return now } + +/* +function handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { + if (event === "$create_alias") { + _alias( + properties["alias"], distinct_id, team_id, + ) + } else if (event === "$identify") { + if (properties["$anon_distinct_id"]) { + _alias( + properties["$anon_distinct_id"], distinct_id, team_id, + ) + } + if (properties["$set"]) { + _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) + } + _set_is_identified(team_id=team_id, distinct_id=distinct_id) + } +} + +function _alias(previous_distinct_id: string, distinct_id: string, team_id: number, retry_if_failed = true): void { + let old_person: Person | null = null + let new_person: Person | null = null + + try { + old_person = Person.objects.get( + team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=previous_distinct_id + ) + except Person.DoesNotExist: + pass + + try: + new_person = Person.objects.get( + team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=distinct_id + ) + except Person.DoesNotExist: + pass + + if old_person and not new_person: + try: + old_person.add_distinct_id(distinct_id) + # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + except IntegrityError: + if retry_if_failed: # run everything again to merge the users if needed + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if not old_person and new_person: + try: + new_person.add_distinct_id(previous_distinct_id) + # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + except IntegrityError: + if retry_if_failed: # run everything again to merge the users if needed + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if not old_person and not new_person: + try: + Person.objects.create( + team_id=team_id, distinct_ids=[str(distinct_id), str(previous_distinct_id)], + ) + # Catch race condition where in between getting and creating, another request already created this user. + except IntegrityError: + if retry_if_failed: + # try once more, probably one of the two persons exists now + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if old_person and new_person and old_person != new_person: + new_person.merge_people([old_person]) +} +*/ + +interface Element { + text: string + tag_name: string + href: string + attr_class: string[] + attr_id: string + nth_child: number + nth_of_type: number + attributes: Record +) + +function _capture_ee( + event_uuid: UUIDT, + person_uuid: UUIDT, + ip: string, + site_url: string, + team_id: number, + event: string, + distinct_id: string, + properties: Properties, + timestamp: DateTime, +): void { + const elements: Record[] = properties.elements + let elements_list: Element[] = [] + if (!isLooselyFalsy(elements)) { + delete properties["$elements"] + elements_list = elements.map((el, index) => ({ + text: el["$el_text"] ? el["$el_text"].slice(0, 400) : null, + tag_name: el["tag_name"], + href: el["attr__href"] ? el["attr__href"].slice(0, 2048) : null, + attr_class: el["attr__class"] ? el["attr__class"].split(" ") : null, + attr_id: el["attr__id"], + nth_child: el["nth_child"], + nth_of_type: el["nth_of_type"], + attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith("attr__"))) + })) + } + + const team = Team.objects.only( + "slack_incoming_webhook", + "event_names", + "event_properties", + "event_names_with_usage", + "event_properties_with_usage", + "anonymize_ips", + ).get(pk=team_id) + + if (!team.anonymize_ips && !("$ip" in properties)) { + properties["$ip"] = ip + } + + store_names_and_properties(team=team, event=event, properties=properties) + + if not Person.objects.distinct_ids_exist(team_id=team_id, distinct_ids=[str(distinct_id)]): + # Catch race condition where in between getting and creating, + # another request already created this user + try: + Person.objects.create(team_id=team_id, distinct_ids=[str(distinct_id)]) + except IntegrityError: + pass + + # # determine create events + create_event( + event_uuid=event_uuid, + event=event, + properties=properties, + timestamp=timestamp, + team=team, + distinct_id=distinct_id, + elements=elements_list, + site_url=site_url, + ) +} diff --git a/src/web/server.ts b/src/web/server.ts index a1923422..58e24667 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -6,8 +6,9 @@ import { KAFKA_EVENTS_WAL } from './topics' import { EventEmitter } from 'events' import { processEventFromKafka } from './process-event' +export const fastifyInstance = fastify() + export function buildFastifyInstance({ DATABASE_URL, EE_ENABLED, KAFKA_HOSTS }: PluginsServerConfig): FastifyInstance { - const fastifyInstance = fastify() fastifyInstance.register(fastifyPostgres, { connectionString: DATABASE_URL, }) diff --git a/src/web/utils.ts b/src/web/utils.ts index 848e3da4..d8431290 100644 --- a/src/web/utils.ts +++ b/src/web/utils.ts @@ -113,3 +113,8 @@ export class UUIDT { return uuidStringify(this.array) } } + +/** Simplistic Python `not` equivalent. */ +export function isLooselyFalsy(value: T): boolean { + return Array.isArray(value) ? !value.length : !value || !Object.keys(value).length +} From 8fd3e427d5b283a97cdbacf9e719d42a6a79bac8 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 3 Dec 2020 17:29:24 +0100 Subject: [PATCH 012/218] Create EventsProcessor with plugins server access --- src/server.ts | 2 +- src/types.ts | 31 ++++ src/web/process-event.ts | 385 ++++++++++++++++++++------------------- src/web/server.ts | 28 +-- 4 files changed, 242 insertions(+), 204 deletions(-) diff --git a/src/server.ts b/src/server.ts index 05d39cbf..7fefaa3f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -72,7 +72,7 @@ export async function startPluginsServer(config: PluginsServerConfig): Promise Promise } } + +export interface EventUsage { + event: string + usage_count: number + volume: number +} + +export interface PropertyUsage { + key: string + usage_count: number + volume: number +} + +export interface Team { + id: number + name: string + anonymize_ips: boolean + api_token: string + app_urls: string[] + completed_snippet_onboarding: boolean + event_names: string[] + event_properties: string[] + event_properties_numerical: string[] + event_names_with_usage: EventUsage[] + event_properties_with_usage: PropertyUsage[] + opt_out_capture: boolean + slack_incoming_webhook: string + session_recording_opt_in: boolean + plugins_opt_in: boolean + ingested_event: boolean +} diff --git a/src/web/process-event.ts b/src/web/process-event.ts index a4a47a8c..c3b3c4bb 100644 --- a/src/web/process-event.ts +++ b/src/web/process-event.ts @@ -1,6 +1,7 @@ import { Message } from 'node-rdkafka' import { DateTime, Duration } from 'luxon' import { isLooselyFalsy, UUIDT } from './utils' +import { PluginsServer, Team } from 'types' export type Data = Record export type Properties = Record @@ -15,219 +16,221 @@ export interface PostHogEvent { sent_at: string } -export function processEventFromKafka(message: Message, commit: () => void): void { - // TODO: time with statsd - const event = JSON.parse(message.value!.toString()) as PostHogEvent - processEventEE( - event.distinct_id, - event.ip, - event.site_url, - event.data, - event.team_id, - DateTime.fromISO(event.now), - event.sent_at ? DateTime.fromISO(event.sent_at) : null - ) - commit() +interface Element { + text: string + tag_name: string + href: string + attr_class: string[] + attr_id: string + nth_child: number + nth_of_type: number + attributes: Record } -function processEventEE( - distinct_id: string, - ip: string, - site_url: string, - data: Data, - team_id: number, - now: DateTime, - sent_at: DateTime | null -): void { - const properties: Properties = data.properties ?? {} - if (data['$set']) { - properties['$set'] = data['$set'] +export class EventsProcessor { + pluginsServer: PluginsServer + + constructor(pluginsServer: PluginsServer) { + this.pluginsServer = pluginsServer } - const person_uuid = new UUIDT() - const event_uuid = new UUIDT() - const ts = handle_timestamp(data, now, sent_at) - // handle_identify_or_alias(data['event'], properties, distinct_id, team_id) - - if (data['event'] === '$snapshot') { - /* - create_session_recording_event( - event_uuid, - team_id, - distinct_id, - properties['$session_id'], - ts, - properties['$snapshot_data'] + processEventFromKafka(message: Message, commit: () => void): void { + // TODO: time with statsd + const event = JSON.parse(message.value!.toString()) as PostHogEvent + this.processEventEE( + event.distinct_id, + event.ip, + event.site_url, + event.data, + event.team_id, + DateTime.fromISO(event.now), + event.sent_at ? DateTime.fromISO(event.sent_at) : null ) - */ - } else { - _capture_ee(event_uuid, person_uuid, ip, site_url, team_id, data['event'], distinct_id, properties, ts) + commit() } -} -function handle_timestamp(data: Data, now: DateTime, sent_at: DateTime | null): DateTime { - if (data['timestamp']) { - if (sent_at) { - // sent_at - timestamp == now - x - // x = now + (timestamp - sent_at) - try { - // timestamp and sent_at must both be in the same format: either both with or both without timezones - // otherwise we can't get a diff to add to now - return now.plus(DateTime.fromISO(data['timestamp']).diff(sent_at)) - } catch (error) { - console.error(error) - } + processEventEE( + distinct_id: string, + ip: string, + site_url: string, + data: Data, + team_id: number, + now: DateTime, + sent_at: DateTime | null + ): void { + const properties: Properties = data.properties ?? {} + if (data['$set']) { + properties['$set'] = data['$set'] + } + + const person_uuid = new UUIDT() + const event_uuid = new UUIDT() + const ts = this.handle_timestamp(data, now, sent_at) + // this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) + + if (data['event'] === '$snapshot') { + /* + this.create_session_recording_event( + event_uuid, + team_id, + distinct_id, + properties['$session_id'], + ts, + properties['$snapshot_data'] + ) + */ + } else { + this._capture_ee(event_uuid, person_uuid, ip, site_url, team_id, data['event'], distinct_id, properties, ts) } - return DateTime.fromISO(data['timestamp']) } - if (!data['offset']) { - return now.minus(Duration.fromMillis(data['offset'])) + + handle_timestamp(data: Data, now: DateTime, sent_at: DateTime | null): DateTime { + if (data['timestamp']) { + if (sent_at) { + // sent_at - timestamp == now - x + // x = now + (timestamp - sent_at) + try { + // timestamp and sent_at must both be in the same format: either both with or both without timezones + // otherwise we can't get a diff to add to now + return now.plus(DateTime.fromISO(data['timestamp']).diff(sent_at)) + } catch (error) { + console.error(error) + } + } + return DateTime.fromISO(data['timestamp']) + } + if (!data['offset']) { + return now.minus(Duration.fromMillis(data['offset'])) + } + return now } - return now -} -/* -function handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { - if (event === "$create_alias") { - _alias( - properties["alias"], distinct_id, team_id, - ) - } else if (event === "$identify") { - if (properties["$anon_distinct_id"]) { + /* + function handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { + if (event === "$create_alias") { _alias( - properties["$anon_distinct_id"], distinct_id, team_id, + properties["alias"], distinct_id, team_id, ) + } else if (event === "$identify") { + if (properties["$anon_distinct_id"]) { + _alias( + properties["$anon_distinct_id"], distinct_id, team_id, + ) + } + if (properties["$set"]) { + _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) + } + _set_is_identified(team_id=team_id, distinct_id=distinct_id) } - if (properties["$set"]) { - _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) - } - _set_is_identified(team_id=team_id, distinct_id=distinct_id) } -} -function _alias(previous_distinct_id: string, distinct_id: string, team_id: number, retry_if_failed = true): void { - let old_person: Person | null = null - let new_person: Person | null = null + function _alias(previous_distinct_id: string, distinct_id: string, team_id: number, retry_if_failed = true): void { + let old_person: Person | null = null + let new_person: Person | null = null - try { - old_person = Person.objects.get( - team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=previous_distinct_id - ) - except Person.DoesNotExist: - pass - - try: - new_person = Person.objects.get( - team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=distinct_id - ) - except Person.DoesNotExist: - pass + try { + old_person = Person.objects.get( + team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=previous_distinct_id + ) + except Person.DoesNotExist: + pass - if old_person and not new_person: try: - old_person.add_distinct_id(distinct_id) - # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id - except IntegrityError: - if retry_if_failed: # run everything again to merge the users if needed - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if not old_person and new_person: - try: - new_person.add_distinct_id(previous_distinct_id) - # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id - except IntegrityError: - if retry_if_failed: # run everything again to merge the users if needed - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if not old_person and not new_person: - try: - Person.objects.create( - team_id=team_id, distinct_ids=[str(distinct_id), str(previous_distinct_id)], + new_person = Person.objects.get( + team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=distinct_id ) - # Catch race condition where in between getting and creating, another request already created this user. - except IntegrityError: - if retry_if_failed: - # try once more, probably one of the two persons exists now - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if old_person and new_person and old_person != new_person: - new_person.merge_people([old_person]) -} -*/ - -interface Element { - text: string - tag_name: string - href: string - attr_class: string[] - attr_id: string - nth_child: number - nth_of_type: number - attributes: Record -) - -function _capture_ee( - event_uuid: UUIDT, - person_uuid: UUIDT, - ip: string, - site_url: string, - team_id: number, - event: string, - distinct_id: string, - properties: Properties, - timestamp: DateTime, -): void { - const elements: Record[] = properties.elements - let elements_list: Element[] = [] - if (!isLooselyFalsy(elements)) { - delete properties["$elements"] - elements_list = elements.map((el, index) => ({ - text: el["$el_text"] ? el["$el_text"].slice(0, 400) : null, - tag_name: el["tag_name"], - href: el["attr__href"] ? el["attr__href"].slice(0, 2048) : null, - attr_class: el["attr__class"] ? el["attr__class"].split(" ") : null, - attr_id: el["attr__id"], - nth_child: el["nth_child"], - nth_of_type: el["nth_of_type"], - attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith("attr__"))) - })) - } + except Person.DoesNotExist: + pass - const team = Team.objects.only( - "slack_incoming_webhook", - "event_names", - "event_properties", - "event_names_with_usage", - "event_properties_with_usage", - "anonymize_ips", - ).get(pk=team_id) - - if (!team.anonymize_ips && !("$ip" in properties)) { - properties["$ip"] = ip + if old_person and not new_person: + try: + old_person.add_distinct_id(distinct_id) + # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + except IntegrityError: + if retry_if_failed: # run everything again to merge the users if needed + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if not old_person and new_person: + try: + new_person.add_distinct_id(previous_distinct_id) + # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + except IntegrityError: + if retry_if_failed: # run everything again to merge the users if needed + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if not old_person and not new_person: + try: + Person.objects.create( + team_id=team_id, distinct_ids=[str(distinct_id), str(previous_distinct_id)], + ) + # Catch race condition where in between getting and creating, another request already created this user. + except IntegrityError: + if retry_if_failed: + # try once more, probably one of the two persons exists now + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if old_person and new_person and old_person != new_person: + new_person.merge_people([old_person]) } + */ + + _capture_ee( + event_uuid: UUIDT, + person_uuid: UUIDT, + ip: string, + site_url: string, + team_id: number, + event: string, + distinct_id: string, + properties: Properties, + timestamp: DateTime, + ): void { + const elements: Record[] = properties.elements + let elements_list: Element[] = [] + if (!isLooselyFalsy(elements)) { + delete properties["$elements"] + elements_list = elements.map((el) => ({ + text: el["$el_text"] ? el["$el_text"].slice(0, 400) : null, + tag_name: el["tag_name"], + href: el["attr__href"] ? el["attr__href"].slice(0, 2048) : null, + attr_class: el["attr__class"] ? el["attr__class"].split(" ") : null, + attr_id: el["attr__id"], + nth_child: el["nth_child"], + nth_of_type: el["nth_of_type"], + attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith("attr__"))) + })) + } + const { rows: [team] }: { rows: Team[] } = await this.pluginsServer.db.query( + "SELECT slack_incoming_webhook, event_names, event_properties, event_names_with_usage, event_properties_with_usage, anonymize_ips FROM posthog_team WHERE id='t'" + ) - store_names_and_properties(team=team, event=event, properties=properties) - - if not Person.objects.distinct_ids_exist(team_id=team_id, distinct_ids=[str(distinct_id)]): - # Catch race condition where in between getting and creating, - # another request already created this user - try: - Person.objects.create(team_id=team_id, distinct_ids=[str(distinct_id)]) - except IntegrityError: - pass + if (!team.anonymize_ips && !("$ip" in properties)) { + properties["$ip"] = ip + } - # # determine create events - create_event( - event_uuid=event_uuid, - event=event, - properties=properties, - timestamp=timestamp, - team=team, - distinct_id=distinct_id, - elements=elements_list, - site_url=site_url, - ) + /// store_names_and_properties(team=team, event=event, properties=properties) + + if not Person.objects.distinct_ids_exist(team_id=team_id, distinct_ids=[str(distinct_id)]): + # Catch race condition where in between getting and creating, + # another request already created this user + try: + Person.objects.create(team_id=team_id, distinct_ids=[str(distinct_id)]) + except IntegrityError: + pass + + # # determine create events + create_event( + event_uuid=event_uuid, + event=event, + properties=properties, + timestamp=timestamp, + team=team, + distinct_id=distinct_id, + elements=elements_list, + site_url=site_url, + ) + } } diff --git a/src/web/server.ts b/src/web/server.ts index 58e24667..ef428728 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -1,32 +1,36 @@ -import { fastify, FastifyInstance, FastifyPluginAsync, FastifyPluginCallback } from 'fastify' +import { fastify, FastifyInstance, FastifyPluginAsync } from 'fastify' import { fastifyPostgres } from 'fastify-postgres' import fastifyKafka from 'fastify-kafka' -import { PluginsServerConfig } from 'types' +import { PluginsServer } from 'types' import { KAFKA_EVENTS_WAL } from './topics' import { EventEmitter } from 'events' -import { processEventFromKafka } from './process-event' +import { EventsProcessor } from './process-event' export const fastifyInstance = fastify() -export function buildFastifyInstance({ DATABASE_URL, EE_ENABLED, KAFKA_HOSTS }: PluginsServerConfig): FastifyInstance { +export function buildFastifyInstance(pluginsServer: PluginsServer): FastifyInstance { fastifyInstance.register(fastifyPostgres, { - connectionString: DATABASE_URL, + connectionString: pluginsServer.DATABASE_URL, }) - if (EE_ENABLED) { + if (pluginsServer.EE_ENABLED) { fastifyInstance.register(fastifyKafka as FastifyPluginAsync, { producer: { dr_cb: true, - 'metadata.broker.list': KAFKA_HOSTS, + 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, }, consumer: { - 'metadata.broker.list': KAFKA_HOSTS, + 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, }, consumerTopicConf: { 'auto.offset.reset': 'earliest', }, }) fastifyInstance.kafka.subscribe([KAFKA_EVENTS_WAL]) - ;((fastifyInstance.kafka as unknown) as EventEmitter).on(KAFKA_EVENTS_WAL, processEventFromKafka) + const eventsProcessor = new EventsProcessor(pluginsServer) + ;((fastifyInstance.kafka as unknown) as EventEmitter).on( + KAFKA_EVENTS_WAL, + eventsProcessor.processEventFromKafka + ) } return fastifyInstance } @@ -36,11 +40,11 @@ export async function stopFastifyInstance(fastifyInstance: FastifyInstance): Pro console.info(`๐Ÿ›‘ Web server cleaned up!`) } -export async function startFastifyInstance(config: PluginsServerConfig): Promise { +export async function startFastifyInstance(pluginsServer: PluginsServer): Promise { console.info(`๐Ÿ‘พ Starting web serverโ€ฆ`) - const fastifyInstance = buildFastifyInstance(config) + const fastifyInstance = buildFastifyInstance(pluginsServer) try { - const address = await fastifyInstance.listen(config.WEB_PORT ?? 3008, config.WEB_HOSTNAME) + const address = await fastifyInstance.listen(pluginsServer.WEB_PORT ?? 3008, pluginsServer.WEB_HOSTNAME) console.info(`โœ… Web server listening on ${address}!`) } catch (e) { console.error(`๐Ÿ›‘ Web server could not start! ${e}`) From a122046dc0873f24fe4931298a3811650a117342 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:47:21 +0100 Subject: [PATCH 013/218] Rework ingestion to separate from Fastify, port over more Django --- src/ingestion/process-event.ts | 310 +++++++++++++++++++++++++++++++ src/{web => ingestion}/topics.ts | 0 src/ingestion/utils.ts | 147 +++++++++++++++ src/web/process-event.ts | 236 ----------------------- src/web/server.ts | 37 +--- src/web/utils.ts | 120 ------------ 6 files changed, 461 insertions(+), 389 deletions(-) create mode 100644 src/ingestion/process-event.ts rename src/{web => ingestion}/topics.ts (100%) create mode 100644 src/ingestion/utils.ts delete mode 100644 src/web/process-event.ts delete mode 100644 src/web/utils.ts diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts new file mode 100644 index 00000000..134cf6ff --- /dev/null +++ b/src/ingestion/process-event.ts @@ -0,0 +1,310 @@ +import { KafkaConsumer } from 'node-rdkafka' +import { DateTime, Duration } from 'luxon' +import { UUID, UUIDT } from './utils' +import { PluginsServer, Data, Properties, Element, Team, Event, Person } from 'types' +import { KAFKA_EVENTS_WAL } from './topics' + +export class EventsProcessor { + pluginsServer: PluginsServer + kafkaConsumer: KafkaConsumer + + constructor(pluginsServer: PluginsServer) { + this.pluginsServer = pluginsServer + this.kafkaConsumer = this.buildKafkaConsumer() + } + + buildKafkaConsumer(): KafkaConsumer { + const kafkaConsumer = new KafkaConsumer( + { + 'metadata.broker.list': this.pluginsServer.KAFKA_HOSTS, + }, + { + 'auto.offset.reset': 'earliest', + } + ) + + kafkaConsumer + .on('ready', () => { + kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) + kafkaConsumer.consume() + console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) + }) + .on('disconnected', () => { + console.info(`๐Ÿ›‘ Kafka consumer disconnected!`) + }) + .on('data', async (message) => { + // TODO: time with statsd + const event = JSON.parse(message.value!.toString()) as Event + await this.process_event_ee( + event.distinct_id, + event.ip, + event.site_url, + event.data, + event.team_id, + DateTime.fromISO(event.now), + event.sent_at ? DateTime.fromISO(event.sent_at) : null + ) + kafkaConsumer.commitMessage(message) + }) + + return kafkaConsumer + } + + /** This must be ran to start consuming events put into Kafka by external web server. */ + connectKafkaConsumer(): void { + console.info(`โฌ Connecting Kafka consumer...`) + this.kafkaConsumer.connect() + } + + async process_event_ee( + distinct_id: string, + ip: string, + site_url: string, + data: Data, + team_id: number, + now: DateTime, + sent_at: DateTime | null + ): void { + const properties: Properties = data.properties ?? {} + if (data['$set']) { + properties['$set'] = data['$set'] + } + + const person_uuid = new UUIDT() + const event_uuid = new UUIDT() + const ts = this.handle_timestamp(data, now, sent_at) + // TODO: this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) + + if (data['event'] === '$snapshot') { + /* TODO: + this.create_session_recording_event( + event_uuid, + team_id, + distinct_id, + properties['$session_id'], + ts, + properties['$snapshot_data'] + ) + */ + } else { + await this._capture_ee( + event_uuid, + person_uuid, + ip, + site_url, + team_id, + data['event'], + distinct_id, + properties, + ts, + sent_at + ) + } + } + + handle_timestamp(data: Data, now: DateTime, sent_at: DateTime | null): DateTime { + if (data['timestamp']) { + if (sent_at) { + // sent_at - timestamp == now - x + // x = now + (timestamp - sent_at) + try { + // timestamp and sent_at must both be in the same format: either both with or both without timezones + // otherwise we can't get a diff to add to now + return now.plus(DateTime.fromISO(data['timestamp']).diff(sent_at)) + } catch (error) { + console.error(error) + } + } + return DateTime.fromISO(data['timestamp']) + } + if (!data['offset']) { + return now.minus(Duration.fromMillis(data['offset'])) + } + return now + } + + /* + function handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { + if (event === "$create_alias") { + _alias( + properties["alias"], distinct_id, team_id, + ) + } else if (event === "$identify") { + if (properties["$anon_distinct_id"]) { + _alias( + properties["$anon_distinct_id"], distinct_id, team_id, + ) + } + if (properties["$set"]) { + _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) + } + _set_is_identified(team_id=team_id, distinct_id=distinct_id) + } + } + + function _alias(previous_distinct_id: string, distinct_id: string, team_id: number, retry_if_failed = true): void { + let old_person: Person | null = null + let new_person: Person | null = null + + try { + old_person = Person.objects.get( + team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=previous_distinct_id + ) + except Person.DoesNotExist: + pass + + try: + new_person = Person.objects.get( + team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=distinct_id + ) + except Person.DoesNotExist: + pass + + if old_person and not new_person: + try: + old_person.add_distinct_id(distinct_id) + # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + except IntegrityError: + if retry_if_failed: # run everything again to merge the users if needed + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if not old_person and new_person: + try: + new_person.add_distinct_id(previous_distinct_id) + # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + except IntegrityError: + if retry_if_failed: # run everything again to merge the users if needed + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if not old_person and not new_person: + try: + Person.objects.create( + team_id=team_id, distinct_ids=[str(distinct_id), str(previous_distinct_id)], + ) + # Catch race condition where in between getting and creating, another request already created this user. + except IntegrityError: + if retry_if_failed: + # try once more, probably one of the two persons exists now + _alias(previous_distinct_id, distinct_id, team_id, False) + return + + if old_person and new_person and old_person != new_person: + new_person.merge_people([old_person]) + } + */ + + async _capture_ee( + event_uuid: UUID, + person_uuid: UUID, + ip: string, + site_url: string, + team_id: number, + event: string, + distinct_id: string, + properties: Properties, + timestamp: DateTime, + sent_at: DateTime + ): void { + const elements: Record[] | undefined = properties['$elements'] + let elements_list: Element[] = [] + if (elements && elements.length) { + delete properties['$elements'] + elements_list = elements.map((el) => ({ + text: el['$el_text'] ? el['$el_text'].slice(0, 400) : null, + tag_name: el['tag_name'], + href: el['attr__href'] ? el['attr__href'].slice(0, 2048) : null, + attr_class: el['attr__class'] ? el['attr__class'].split(' ') : null, + attr_id: el['attr__id'], + nth_child: el['nth_child'], + nth_of_type: el['nth_of_type'], + attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith('attr__'))), + })) + } + const { + rows: [team], + }: { + rows: Team[] + } = await this.pluginsServer.db.query( + 'SELECT slack_incoming_webhook, event_names, event_properties, event_names_with_usage, event_properties_with_usage, anonymize_ips FROM posthog_team WHERE id = $1', + [team_id] + ) + + if (!team.anonymize_ips && !('$ip' in properties)) { + properties['$ip'] = ip + } + + // TODO: store_names_and_properties(team=team, event=event, properties=properties) + + const { + rows: [{ pdiCount }], + }: { + rows: { pdiCount: number }[] + } = await this.pluginsServer.db.query( + 'SELECT COUNT(*) AS pdiCount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', + [team_id, distinct_id] + ) + + if (!pdiCount) { + // Catch race condition where in between getting and creating, another request already created this user + try { + const { + rows: [personCreated], + }: { + rows: Person[] + } = await this.pluginsServer.db.query( + 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6)', + [sent_at.toISO(), {}, team_id, null, false, person_uuid.toString()] + ) + await this.pluginsServer.db.query( + 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3)', + [distinct_id, personCreated.id, team_id] + ) + } catch {} + } + + this.create_event( + event_uuid, + event, + team, + distinct_id, + properties, + timestamp, + elements_list, + ) + } + + create_event( + event_uuid: UUID, + event: string, + team: Team, + distinct_id: string, + properties?: Properties, + timestamp?: DateTime | string, + elements?: Element[] + ): void { + timestamp ??= DateTime.utc() + + // ClickHouse specific formatting + timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() + + let elements_chain = elements && elements.length ? elements_to_string(elements=elements) : '' + + pb_event = events_pb2.Event() + pb_event.uuid = str(event_uuid) + pb_event.event = event + pb_event.properties = json.dumps(properties ?? {}) + pb_event.timestamp = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") + pb_event.team_id = team.pk + pb_event.distinct_id = str(distinct_id) + pb_event.elements_chain = elements_chain + pb_event.created_at = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") + + p = ClickhouseProducer() + + p.produce_proto(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=pb_event) + + return str(event_uuid) + } +} diff --git a/src/web/topics.ts b/src/ingestion/topics.ts similarity index 100% rename from src/web/topics.ts rename to src/ingestion/topics.ts diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts new file mode 100644 index 00000000..86cb9b55 --- /dev/null +++ b/src/ingestion/utils.ts @@ -0,0 +1,147 @@ +import { randomBytes } from 'crypto' +import { DateTime } from 'luxon' + +/** LUT of byte value to hexadecimal representation. For UUID stringification. */ +const byteToHex: string[] = [] +for (let i = 0; i < 256; i++) { + byteToHex.push((i + 0x100).toString(16).substr(1)) +} + +export class UUID { + static validateString(candidate: string): void { + if (!candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i)) { + throw new Error( + 'String does not match format XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX (where each X is a hexadecimal character)!' + ) + } + } + + array: Uint8Array + + constructor(candidate: string | Uint8Array | Buffer) { + if (candidate instanceof Uint8Array) { + if (candidate.byteLength !== 16) { + throw new Error(`UUID must be built from exactly 16 bytes, but you provided ${candidate.byteLength}!`) + } + this.array = new Uint8Array(candidate) + } else { + candidate = candidate.trim() + UUID.validateString(candidate) + this.array = new Uint8Array(16) + const characters = Array.from(candidate).filter((character) => character !== '-') + for (let i = 0; i < characters.length; i += 2) { + this.array[i / 2] = parseInt(characters[i] + characters[i + 1], 16) + } + } + } + + /** Convert to 128-bit BigInt. */ + toNumber(): BigInt { + let value = 0n + for (const byte of this.array) { + value <<= 8n + value += BigInt(byte) + } + return value + } + + /** + * Convert to string format of the form: + * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + * Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js + */ + toString(): string { + // Note: Be careful editing this code! It's been tuned for performance and works in ways you may not expect. + // See https://github.com/uuidjs/uuid/pull/434 + const arr = this.array + return ( + byteToHex[arr[0]] + + byteToHex[arr[1]] + + byteToHex[arr[2]] + + byteToHex[arr[3]] + + '-' + + byteToHex[arr[4]] + + byteToHex[arr[5]] + + '-' + + byteToHex[arr[6]] + + byteToHex[arr[7]] + + '-' + + byteToHex[arr[8]] + + byteToHex[arr[9]] + + '-' + + byteToHex[arr[10]] + + byteToHex[arr[11]] + + byteToHex[arr[12]] + + byteToHex[arr[13]] + + byteToHex[arr[14]] + + byteToHex[arr[15]] + ).toLowerCase() + } +} + +/** + * UUID (mostly) sortable by generation time. + * + * This doesn't adhere to any official UUID version spec, but it is superior as a primary key: + * to incremented integers (as they can reveal sensitive business information about usage volumes and patterns), + * to UUID v4 (as the complete randomness of v4 makes its indexing performance suboptimal), + * and to UUID v1 (as despite being time-based it can't be used practically for sorting by generation time). + * + * Order can be messed up if system Unix time is changed or if more than 65ย 536 IDs are generated per millisecond + * (that's over 5 trillion events per day), but it should be largely safe to assume that these are time-sortable. + * + * Anatomy: + * - 6 bytes - Unix time milliseconds unsigned integer + * - 2 bytes - autoincremented series unsigned integer (per millisecond, rolls over to 0 after reaching 65 535 UUIDs in one ms) + * - 8 bytes - securely random gibberish + * + * Loosely based on [Segment's KSUID](https://github.com/segmentio/ksuid) and + * on [Twitter's snowflake ID](https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html). + * Ported from the PostHog Django app. + */ +export class UUIDT extends UUID { + static currentSeriesPerMs: Map = new Map() + + /** Get per-millisecond series integer in range [0-65536). */ + static getSeries(unixTimeMs: number): number { + const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) + if (UUIDT.currentSeriesPerMs.size > 10_000) { + // Clear class dict periodically + UUIDT.currentSeriesPerMs.clear() + } + const nextSeries = typeof series === 'number' ? (series + 1) % 65_536 : 0 + UUIDT.currentSeriesPerMs.set(unixTimeMs, nextSeries) + return nextSeries + } + + constructor(unixTimeMs?: number) { + if (!unixTimeMs) { + unixTimeMs = DateTime.utc().toMillis() + } + let series = UUIDT.getSeries(unixTimeMs) + // 64 bits (8 bytes) total + const array = new Uint8Array(16) + // 48 bits for time, WILL FAIL in 10 895 CE + // XXXXXXXX-XXXX-****-****-************ + // TODO: why the hell are the two leftmost octets (four hexadecimal chars) always zero? + for (let i = 5; i >= 0; i--) { + array[i] = unixTimeMs & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits + unixTimeMs >>>= 8 // remove these last 8 binary digits + } + // 16 bits for series + // ********-****-XXXX-****-************ + for (let i = 7; i >= 6; i--) { + array[i] = series & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits + series >>>= 8 // remove these last 8 binary digits + } + // 64 bits for random gibberish + // ********-****-****-XXXX-XXXXXXXXXXXX + array.set(randomBytes(8), 8) + super(array) + } +} + +/** Simplistic Python `not` equivalent. */ +export function isLooselyFalsy(value: any): boolean { + return Array.isArray(value) ? !value.length : !value || !Object.keys(value).length +} diff --git a/src/web/process-event.ts b/src/web/process-event.ts deleted file mode 100644 index c3b3c4bb..00000000 --- a/src/web/process-event.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { Message } from 'node-rdkafka' -import { DateTime, Duration } from 'luxon' -import { isLooselyFalsy, UUIDT } from './utils' -import { PluginsServer, Team } from 'types' - -export type Data = Record -export type Properties = Record - -export interface PostHogEvent { - distinct_id: string - ip: string - site_url: string - data: Data - team_id: number - now: string - sent_at: string -} - -interface Element { - text: string - tag_name: string - href: string - attr_class: string[] - attr_id: string - nth_child: number - nth_of_type: number - attributes: Record -} - -export class EventsProcessor { - pluginsServer: PluginsServer - - constructor(pluginsServer: PluginsServer) { - this.pluginsServer = pluginsServer - } - - processEventFromKafka(message: Message, commit: () => void): void { - // TODO: time with statsd - const event = JSON.parse(message.value!.toString()) as PostHogEvent - this.processEventEE( - event.distinct_id, - event.ip, - event.site_url, - event.data, - event.team_id, - DateTime.fromISO(event.now), - event.sent_at ? DateTime.fromISO(event.sent_at) : null - ) - commit() - } - - processEventEE( - distinct_id: string, - ip: string, - site_url: string, - data: Data, - team_id: number, - now: DateTime, - sent_at: DateTime | null - ): void { - const properties: Properties = data.properties ?? {} - if (data['$set']) { - properties['$set'] = data['$set'] - } - - const person_uuid = new UUIDT() - const event_uuid = new UUIDT() - const ts = this.handle_timestamp(data, now, sent_at) - // this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) - - if (data['event'] === '$snapshot') { - /* - this.create_session_recording_event( - event_uuid, - team_id, - distinct_id, - properties['$session_id'], - ts, - properties['$snapshot_data'] - ) - */ - } else { - this._capture_ee(event_uuid, person_uuid, ip, site_url, team_id, data['event'], distinct_id, properties, ts) - } - } - - handle_timestamp(data: Data, now: DateTime, sent_at: DateTime | null): DateTime { - if (data['timestamp']) { - if (sent_at) { - // sent_at - timestamp == now - x - // x = now + (timestamp - sent_at) - try { - // timestamp and sent_at must both be in the same format: either both with or both without timezones - // otherwise we can't get a diff to add to now - return now.plus(DateTime.fromISO(data['timestamp']).diff(sent_at)) - } catch (error) { - console.error(error) - } - } - return DateTime.fromISO(data['timestamp']) - } - if (!data['offset']) { - return now.minus(Duration.fromMillis(data['offset'])) - } - return now - } - - /* - function handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { - if (event === "$create_alias") { - _alias( - properties["alias"], distinct_id, team_id, - ) - } else if (event === "$identify") { - if (properties["$anon_distinct_id"]) { - _alias( - properties["$anon_distinct_id"], distinct_id, team_id, - ) - } - if (properties["$set"]) { - _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) - } - _set_is_identified(team_id=team_id, distinct_id=distinct_id) - } - } - - function _alias(previous_distinct_id: string, distinct_id: string, team_id: number, retry_if_failed = true): void { - let old_person: Person | null = null - let new_person: Person | null = null - - try { - old_person = Person.objects.get( - team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=previous_distinct_id - ) - except Person.DoesNotExist: - pass - - try: - new_person = Person.objects.get( - team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=distinct_id - ) - except Person.DoesNotExist: - pass - - if old_person and not new_person: - try: - old_person.add_distinct_id(distinct_id) - # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id - except IntegrityError: - if retry_if_failed: # run everything again to merge the users if needed - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if not old_person and new_person: - try: - new_person.add_distinct_id(previous_distinct_id) - # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id - except IntegrityError: - if retry_if_failed: # run everything again to merge the users if needed - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if not old_person and not new_person: - try: - Person.objects.create( - team_id=team_id, distinct_ids=[str(distinct_id), str(previous_distinct_id)], - ) - # Catch race condition where in between getting and creating, another request already created this user. - except IntegrityError: - if retry_if_failed: - # try once more, probably one of the two persons exists now - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if old_person and new_person and old_person != new_person: - new_person.merge_people([old_person]) - } - */ - - _capture_ee( - event_uuid: UUIDT, - person_uuid: UUIDT, - ip: string, - site_url: string, - team_id: number, - event: string, - distinct_id: string, - properties: Properties, - timestamp: DateTime, - ): void { - const elements: Record[] = properties.elements - let elements_list: Element[] = [] - if (!isLooselyFalsy(elements)) { - delete properties["$elements"] - elements_list = elements.map((el) => ({ - text: el["$el_text"] ? el["$el_text"].slice(0, 400) : null, - tag_name: el["tag_name"], - href: el["attr__href"] ? el["attr__href"].slice(0, 2048) : null, - attr_class: el["attr__class"] ? el["attr__class"].split(" ") : null, - attr_id: el["attr__id"], - nth_child: el["nth_child"], - nth_of_type: el["nth_of_type"], - attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith("attr__"))) - })) - } - const { rows: [team] }: { rows: Team[] } = await this.pluginsServer.db.query( - "SELECT slack_incoming_webhook, event_names, event_properties, event_names_with_usage, event_properties_with_usage, anonymize_ips FROM posthog_team WHERE id='t'" - ) - - if (!team.anonymize_ips && !("$ip" in properties)) { - properties["$ip"] = ip - } - - /// store_names_and_properties(team=team, event=event, properties=properties) - - if not Person.objects.distinct_ids_exist(team_id=team_id, distinct_ids=[str(distinct_id)]): - # Catch race condition where in between getting and creating, - # another request already created this user - try: - Person.objects.create(team_id=team_id, distinct_ids=[str(distinct_id)]) - except IntegrityError: - pass - - # # determine create events - create_event( - event_uuid=event_uuid, - event=event, - properties=properties, - timestamp=timestamp, - team=team, - distinct_id=distinct_id, - elements=elements_list, - site_url=site_url, - ) - } -} diff --git a/src/web/server.ts b/src/web/server.ts index ef428728..2b8fd217 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -1,37 +1,8 @@ -import { fastify, FastifyInstance, FastifyPluginAsync } from 'fastify' -import { fastifyPostgres } from 'fastify-postgres' -import fastifyKafka from 'fastify-kafka' +import { fastify, FastifyInstance } from 'fastify' import { PluginsServer } from 'types' -import { KAFKA_EVENTS_WAL } from './topics' -import { EventEmitter } from 'events' -import { EventsProcessor } from './process-event' -export const fastifyInstance = fastify() - -export function buildFastifyInstance(pluginsServer: PluginsServer): FastifyInstance { - fastifyInstance.register(fastifyPostgres, { - connectionString: pluginsServer.DATABASE_URL, - }) - if (pluginsServer.EE_ENABLED) { - fastifyInstance.register(fastifyKafka as FastifyPluginAsync, { - producer: { - dr_cb: true, - 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, - }, - consumer: { - 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, - }, - consumerTopicConf: { - 'auto.offset.reset': 'earliest', - }, - }) - fastifyInstance.kafka.subscribe([KAFKA_EVENTS_WAL]) - const eventsProcessor = new EventsProcessor(pluginsServer) - ;((fastifyInstance.kafka as unknown) as EventEmitter).on( - KAFKA_EVENTS_WAL, - eventsProcessor.processEventFromKafka - ) - } +export function buildFastifyInstance(): FastifyInstance { + const fastifyInstance = fastify() return fastifyInstance } @@ -42,7 +13,7 @@ export async function stopFastifyInstance(fastifyInstance: FastifyInstance): Pro export async function startFastifyInstance(pluginsServer: PluginsServer): Promise { console.info(`๐Ÿ‘พ Starting web serverโ€ฆ`) - const fastifyInstance = buildFastifyInstance(pluginsServer) + const fastifyInstance = buildFastifyInstance() try { const address = await fastifyInstance.listen(pluginsServer.WEB_PORT ?? 3008, pluginsServer.WEB_HOSTNAME) console.info(`โœ… Web server listening on ${address}!`) diff --git a/src/web/utils.ts b/src/web/utils.ts deleted file mode 100644 index d8431290..00000000 --- a/src/web/utils.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { randomBytes } from 'crypto' -import { DateTime } from 'luxon'; - -const byteToHex: string[] = [] -for (let i = 0; i < 256; i++) { - byteToHex.push((i + 0x100).toString(16).substr(1)); -} - -/** - * Convert array of 16 byte values to UUID string format of the form: - * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX - * Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js - */ -function uuidStringify(arr: Uint8Array) { - // Note: Be careful editing this code! It's been tuned for performance and works in ways you may not expect. - // See https://github.com/uuidjs/uuid/pull/434 - return ( - byteToHex[arr[0]] + - byteToHex[arr[1]] + - byteToHex[arr[2]] + - byteToHex[arr[3]] + - '-' + - byteToHex[arr[4]] + - byteToHex[arr[5]] + - '-' + - byteToHex[arr[6]] + - byteToHex[arr[7]] + - '-' + - byteToHex[arr[8]] + - byteToHex[arr[9]] + - '-' + - byteToHex[arr[10]] + - byteToHex[arr[11]] + - byteToHex[arr[12]] + - byteToHex[arr[13]] + - byteToHex[arr[14]] + - byteToHex[arr[15]] - ).toLowerCase() -} - -/** - * UUID (mostly) sortable by generation time. - * - * This doesn't adhere to any official UUID version spec, but it is superior as a primary key: - * to incremented integers (as they can reveal sensitive business information about usage volumes and patterns), - * to UUID v4 (as the complete randomness of v4 makes its indexing performance suboptimal), - * and to UUID v1 (as despite being time-based it can't be used practically for sorting by generation time). - * - * Order can be messed up if system clock is changed or if more than 65ย 536 IDs are generated per millisecond - * (that's over 5 trillion events per day), but it should be largely safe to assume that these are time-sortable. - * - * Anatomy: - * - 6 bytes - Unix time milliseconds unsigned integer - * - 2 bytes - autoincremented series unsigned integer (per millisecond, rolls over to 0 after reaching 65 535 UUIDs in one ms) - * - 8 bytes - securely random gibberish - * - * Loosely based on [Segment's KSUID](https://github.com/segmentio/ksuid) and - * on [Twitter's snowflake ID](https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html). - * Ported from the PostHog Django app. - */ -export class UUIDT { - static currentSeriesPerMs: Map = new Map() - - /** Get per-millisecond series integer in range [0-65536). */ - static getSeries(unixTimeMs: number): number { - const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) - if (UUIDT.currentSeriesPerMs.size > 10_000) { - // Clear class dict periodically - UUIDT.currentSeriesPerMs.clear() - } - const nextSeries = typeof series === 'number' ? (series + 1) % 65_536 : 0 - UUIDT.currentSeriesPerMs.set(unixTimeMs, nextSeries) - return nextSeries - } - - array: Uint8Array - - constructor(unixTimeMs?: number) { - if (!unixTimeMs) { - unixTimeMs = DateTime.utc().toMillis() - } - let series = UUIDT.getSeries(unixTimeMs) - // 64 bits (8 bytes) total - this.array = new Uint8Array(16) - // 48 bits for time, WILL FAIL in 10 895 CE - // XXXXXXXX-XXXX-****-****-************ - // TODO: why the hell are the two leftmost octets (four hexadecimal chars) always zero? - for (let i = 5; i >= 0; i--) { - this.array[i] = unixTimeMs & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits - unixTimeMs >>= 8 // remove these last 8 binary digits - } - // 16 bits for series - // ********-****-XXXX-****-************ - for (let i = 7; i >= 6; i--) { - this.array[i] = series & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits - series >>= 8 // remove these last 8 binary digits - } - // 64 bits for random gibberish - // ********-****-****-XXXX-XXXXXXXXXXXX - this.array.set(randomBytes(8), 8) - } - - toNumber(): number { - let value = 0 - for (const byte of this.array) { - value <<= 8 - value += byte - } - return value - } - - toString(): string { - return uuidStringify(this.array) - } -} - -/** Simplistic Python `not` equivalent. */ -export function isLooselyFalsy(value: T): boolean { - return Array.isArray(value) ? !value.length : !value || !Object.keys(value).length -} From fca0ef9840bbc32b6b3839ec9fb5ed2eb77fb1b5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:47:39 +0100 Subject: [PATCH 014/218] Update utils.test.ts --- src/__tests__/ingestion/utils.test.ts | 119 ++++++++++++++++++++++++++ src/__tests__/web/utils.test.ts | 12 --- 2 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 src/__tests__/ingestion/utils.test.ts delete mode 100644 src/__tests__/web/utils.test.ts diff --git a/src/__tests__/ingestion/utils.test.ts b/src/__tests__/ingestion/utils.test.ts new file mode 100644 index 00000000..033c82b3 --- /dev/null +++ b/src/__tests__/ingestion/utils.test.ts @@ -0,0 +1,119 @@ +import { randomBytes } from 'crypto' +import { UUID, UUIDT } from '../../ingestion/utils' + +describe('UUID', () => { + describe('#constructor', () => { + it('works with a valid string', () => { + const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') + expect(uuid.array).toStrictEqual( + new Uint8Array([ + 0x99, + 0xab, + 0xcd, + 0xef, + 0x12, + 0x34, + 0x43, + 0x21, + 0, + 0, + 0xdc, + 0xba, + 0x87, + 0x65, + 0x43, + 0x21, + ]) + ) + }) + + it('throws on an invalid string', () => { + expect(() => new UUID('99aBcDeF-1234-4321-WXyz-dcba87654321')).toThrow() // "WXyz" are not hexadecimal + expect(() => new UUID('99aBcDeF123443210000dcba87654321')).toThrow() // lack of hyphens + expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba87654321A')).toThrow() // one character too many + expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba8765432')).toThrow() // one character too few + expect(() => new UUID('')).toThrow() // empty string + }) + + it('works with a Uint8Array', () => { + for (let i = 0; i < 10; i++) { + const uuid = new UUID( + new Uint8Array([ + 0x99, + 0xab, + 0xcd, + 0xef, + 0x12, + 0x34, + 0x43, + 0x21, + 0, + 0, + 0xdc, + 0xba, + 0x87, + 0x65, + 0x43, + 0x21, + ]) + ) + expect(uuid.array).toStrictEqual( + new Uint8Array([ + 0x99, + 0xab, + 0xcd, + 0xef, + 0x12, + 0x34, + 0x43, + 0x21, + 0, + 0, + 0xdc, + 0xba, + 0x87, + 0x65, + 0x43, + 0x21, + ]) + ) + } + }) + + it('works with a random buffer', () => { + for (let i = 0; i < 10; i++) { + const uuid = new UUID(randomBytes(16)) + expect(uuid.array).toHaveLength(16) + } + }) + }) + + describe('#toNumber', () => { + it('returns the right big integer', () => { + const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') + expect(uuid.toNumber()).toStrictEqual(0x99abcdef123443210000dcba87654321n) + }) + }) + + describe('#toString', () => { + it('returns the right string', () => { + const original = '99aBcDeF-1234-4321-0000-dcba87654321' + const uuid = new UUID(original) + const uuidString = uuid.toString() + // 32 hexadecimal digits + 4 hyphens + expect(uuidString).toHaveLength(36) + expect(uuidString).toStrictEqual(original.toLowerCase()) + }) + }) +}) + +describe('UUIDT', () => { + it('is well-formed', () => { + const uuidt = new UUIDT() + const uuidtString = uuidt.toString() + // UTC timestamp matching (roughly) + expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8)) + // series matching + expect(uuidtString.slice(14, 18)).toEqual('0000') + }) +}) diff --git a/src/__tests__/web/utils.test.ts b/src/__tests__/web/utils.test.ts deleted file mode 100644 index 49992fe0..00000000 --- a/src/__tests__/web/utils.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { UUIDT } from '../../web/utils' - -test('UUIDT', async () => { - const uuidt = new UUIDT() - const uuidtString = uuidt.toString() - // 32 hexadecimal digits + 4 hyphens - expect(uuidtString).toHaveLength(36) - // UTC timestamp matching (roughly) - expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8)) - // series matching - expect(uuidtString.slice(14, 18)).toEqual('0000') -}) From bf88f22aef4d18b23f973ed22be9ca80059df56c Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:47:56 +0100 Subject: [PATCH 015/218] Remove fastify-kafka and fastify-postgres --- package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/package.json b/package.json index 3ce37873..e6dc2799 100644 --- a/package.json +++ b/package.json @@ -28,8 +28,6 @@ "@types/luxon": "^1.25.0", "adm-zip": "^0.4.16", "fastify": "^3.8.0", - "fastify-kafka": "^0.3.0", - "fastify-postgres": "^3.3.0", "ioredis": "^4.19.2", "luxon": "^1.25.0", "node-fetch": "^2.6.1", From 14fdf2cb0c69cdc45780eb085155856afa2c17db Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:48:11 +0100 Subject: [PATCH 016/218] Add protobuf --- package.json | 1 + src/ingestion/idl/element.proto | 19 +++ src/ingestion/idl/events.proto | 15 ++ .../idl/google/protobuf/timestamp.proto | 147 ++++++++++++++++++ src/ingestion/idl/omni_person.proto | 12 ++ src/ingestion/idl/person.proto | 9 ++ 6 files changed, 203 insertions(+) create mode 100644 src/ingestion/idl/element.proto create mode 100644 src/ingestion/idl/events.proto create mode 100644 src/ingestion/idl/google/protobuf/timestamp.proto create mode 100644 src/ingestion/idl/omni_person.proto create mode 100644 src/ingestion/idl/person.proto diff --git a/package.json b/package.json index e6dc2799..8fc41d1d 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "pg": "^8.4.2", "posthog-js-lite": "^0.0.5", "posthog-plugins": "^0.2.3", + "protobufjs": "^6.10.2", "tar-stream": "^2.1.4", "uuid": "^8.3.1", "vm2": "^3.9.2", diff --git a/src/ingestion/idl/element.proto b/src/ingestion/idl/element.proto new file mode 100644 index 00000000..a96a0c6b --- /dev/null +++ b/src/ingestion/idl/element.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +import "google/protobuf/timestamp.proto"; + +message Element { + string uuid = 1; + string event_uuid = 2; + string text = 3; + string tag_name = 4; + string href = 5; + string attr_id = 6; + repeated string attr_class = 7; + uint64 nth_child = 8; + uint64 nth_of_type = 9; + string attributes = 10; + uint64 order = 11; + uint64 team_id = 12; + google.protobuf.Timestamp created_at = 13; + string elements_hash = 14; +} \ No newline at end of file diff --git a/src/ingestion/idl/events.proto b/src/ingestion/idl/events.proto new file mode 100644 index 00000000..afb1ce07 --- /dev/null +++ b/src/ingestion/idl/events.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; +import "google/protobuf/timestamp.proto"; + +message Event { + string uuid = 1; + string event = 2; + string properties = 3; + string timestamp = 4; + uint64 team_id = 5; + string distinct_id = 6; + string created_at = 7; + string elements_chain = 8; + google.protobuf.Timestamp proto_created_at = 9; + google.protobuf.Timestamp proto_timestamp = 10; +} \ No newline at end of file diff --git a/src/ingestion/idl/google/protobuf/timestamp.proto b/src/ingestion/idl/google/protobuf/timestamp.proto new file mode 100644 index 00000000..9c4c75cc --- /dev/null +++ b/src/ingestion/idl/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} \ No newline at end of file diff --git a/src/ingestion/idl/omni_person.proto b/src/ingestion/idl/omni_person.proto new file mode 100644 index 00000000..d327e1ac --- /dev/null +++ b/src/ingestion/idl/omni_person.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +import "google/protobuf/timestamp.proto"; + +message Person { + string uuid = 1; + string event_uuid = 2; + uint64 team_id = 3; + string distinct_id = 4; + string properties = 5; + bool is_identified = 6; + google.protobuf.Timestamp ts = 7; +} diff --git a/src/ingestion/idl/person.proto b/src/ingestion/idl/person.proto new file mode 100644 index 00000000..221f5264 --- /dev/null +++ b/src/ingestion/idl/person.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +import "google/protobuf/timestamp.proto"; + +message Person { + string uuid = 1; + google.protobuf.Timestamp created_at = 2; + string properties = 3; + bool is_identified = 4; +} From d2f42dd0030f918b4e23ed1a95be611c9d885fe4 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:48:33 +0100 Subject: [PATCH 017/218] Add types --- src/types.ts | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/types.ts b/src/types.ts index 838c930f..e8bac29b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,6 +2,7 @@ import { Pool } from 'pg' import { Redis } from 'ioredis' import { PluginEvent, PluginAttachment, PluginConfigSchema } from 'posthog-plugins' import { VM, VMScript } from 'vm2' +import { DateTime } from 'luxon' export interface PluginsServerConfig { CELERY_DEFAULT_QUEUE: string @@ -13,8 +14,8 @@ export interface PluginsServerConfig { BASE_DIR: string PLUGINS_RELOAD_PUBSUB_CHANNEL: string DISABLE_WEB: boolean - WEB_PORT?: number - WEB_HOSTNAME?: string + WEB_PORT: number + WEB_HOSTNAME: string } export interface PluginsServer extends PluginsServerConfig { @@ -121,3 +122,40 @@ export interface Team { plugins_opt_in: boolean ingested_event: boolean } + +export type Data = Record +export type Properties = Record + +export interface Event { + id: number + distinct_id: string + ip: string + site_url: string + data: Data + team_id: number + now: string + sent_at: string +} + +export interface Element { + text: string + tag_name: string + href: string + attr_class: string[] + attr_id: string + nth_child: number + nth_of_type: number + attributes: Record +} + +export type User = Record // not really typed as not needed so far + +export interface Person { + id: number + created_at: DateTime + team: Team + properties: Properties + is_user: User + is_identified: boolean + uuid: string +} From dfb1454574c9c63160ae7174e8c13043de6e45aa Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:48:39 +0100 Subject: [PATCH 018/218] Update yarn.lock --- yarn.lock | 116 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 88 insertions(+), 28 deletions(-) diff --git a/yarn.lock b/yarn.lock index 068906b7..7dfed766 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1183,6 +1183,59 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha1-m4sMxmPWaafY9vXQiToU00jzD78= + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha1-NVy8mLr61ZePntCV85diHx0Ga3A= + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU= + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E= + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik= + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha1-bMKyDFya1q0NzP0hynZz2Nf79o0= + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q= + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= + "@rollup/plugin-json@^4.1.0": version "4.1.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-json/-/plugin-json-4.1.0.tgz#54e09867ae6963c593844d8bd7a9c718294496f3" @@ -1319,6 +1372,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/long@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" + integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== + "@types/luxon@^1.25.0": version "1.25.0" resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-1.25.0.tgz#3d6fe591fac874f48dd225cb5660b2b785a21a05" @@ -1344,6 +1402,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.5.tgz#e92d3b8f76583efa26c1a63a21c9d3c1143daa29" integrity sha512-H5Wn24s/ZOukBmDn03nnGTp18A60ny9AmCwnEcgJiTgSGsCO7k+NWP7zjCCbhlcnVCoI+co52dUAt9GMhOSULw== +"@types/node@^13.7.0": + version "13.13.34" + resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.34.tgz#c9300a1b6560d90817fb2bba650e250116a575f9" + integrity sha512-g8D1HF2dMDKYSDl5+79izRwRgNPsSynmWMbj50mj7GZ0b7Lv4p8EmZjbo3h0h+6iLr6YmVz9VnF6XVZ3O6V1Ug== + "@types/node@^14.14.6": version "14.14.6" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.6.tgz#146d3da57b3c636cc0d1769396ce1cfa8991147f" @@ -3118,33 +3181,6 @@ fastify-error@^0.2.0: resolved "https://registry.yarnpkg.com/fastify-error/-/fastify-error-0.2.0.tgz#9a1c28d4f42b6259e7a549671c8e5e2d85660634" integrity sha512-zabxsBatj59ROG0fhP36zNdc5Q1/eYeH9oSF9uvfrurZf8/JKfrJbMcIGrLpLWcf89rS6L91RHWm20A/X85hcA== -fastify-kafka@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/fastify-kafka/-/fastify-kafka-0.3.0.tgz#3fc83b7d0a66370c5d80704acb65bbdb4a9c0644" - integrity sha512-HtzM89XrW6JCQQMWYbzQHgKOr5bLFxqdAAA2IbJ6Q8bueLhaWuTjvfHgWzhzlCOy3rbSQNo+bi8GGpz2KZTDEw== - dependencies: - fastify-plugin "^2.0.0" - node-rdkafka "^2.8.1" - -fastify-plugin@^2.0.0: - version "2.3.4" - resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-2.3.4.tgz#b17abdc36a97877d88101fb86ad8a07f2c07de87" - integrity sha512-I+Oaj6p9oiRozbam30sh39BiuiqBda7yK2nmSPVwDCfIBlKnT8YB3MY+pRQc2Fcd07bf6KPGklHJaQ2Qu81TYQ== - dependencies: - semver "^7.3.2" - -fastify-plugin@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/fastify-plugin/-/fastify-plugin-3.0.0.tgz#cf1b8c8098e3b5a7c8c30e6aeb06903370c054ca" - integrity sha512-ZdCvKEEd92DNLps5n0v231Bha8bkz1DjnPP/aEz37rz/q42Z5JVLmgnqR4DYuNn3NXAO3IDCPyRvgvxtJ4Ym4w== - -fastify-postgres@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/fastify-postgres/-/fastify-postgres-3.3.0.tgz#e23e86957ac826757999b92db98120e9edaf796c" - integrity sha512-B0yoveN1Oo9oLM4phFl5yscmj1Lpq1lD9YXXUIDRAsd0eeJdvphT+bEHARHFnoSQuFr5QGViqtpPIrRDZQxMIA== - dependencies: - fastify-plugin "^3.0.0" - fastify-warning@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/fastify-warning/-/fastify-warning-0.2.0.tgz#e717776026a4493dc9a2befa44db6d17f618008f" @@ -4779,6 +4815,11 @@ long-timeout@0.1.1: resolved "https://registry.yarnpkg.com/long-timeout/-/long-timeout-0.1.1.tgz#9721d788b47e0bcb5a24c2e2bee1a0da55dab514" integrity sha1-lyHXiLR+C8taJMLivuGg2lXatRQ= +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -5048,7 +5089,7 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" -node-rdkafka@^2.8.1, node-rdkafka@^2.9.1: +node-rdkafka@^2.9.1: version "2.9.1" resolved "https://registry.yarnpkg.com/node-rdkafka/-/node-rdkafka-2.9.1.tgz#ac711bc7845ccb1de87391aec7f27f683d265149" integrity sha512-C5EVDZlDG+5D8KXiz2zKwEiLWIGW5Z1mkVFRzp13T4mrbXz+ESyjrDSLIj7aoUIi5+T10H9p1wwLZJBh9ivjLg== @@ -5627,6 +5668,25 @@ prop-types@^15.7.2: object-assign "^4.1.1" react-is "^16.8.1" +protobufjs@^6.10.2: + version "6.10.2" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.10.2.tgz#b9cb6bd8ec8f87514592ba3fdfd28e93f33a469b" + integrity sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.1" + "@types/node" "^13.7.0" + long "^4.0.0" + proxy-addr@^2.0.5: version "2.0.6" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.6.tgz#fdc2336505447d3f2f2c638ed272caf614bbb2bf" From 028c892f2165a81bd39702c438608785677fc465 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:48:56 +0100 Subject: [PATCH 019/218] Improve info --- src/celery/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/celery/worker.ts b/src/celery/worker.ts index 2fa7d136..144efa01 100644 --- a/src/celery/worker.ts +++ b/src/celery/worker.ts @@ -78,7 +78,7 @@ export class Worker extends Base { return () => { const result = this.broker.subscribe(queue, onMessage) - console.info(`โœ… Celery worker subscribed to ${Object.keys(this.handlers).join(', ')}`) + console.info(`โœ… Celery worker subscribed to ${Object.keys(this.handlers).join(', ')}!`) return result } } From 239f90411b7141cc583f66d6218e3a7554613744 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 09:49:02 +0100 Subject: [PATCH 020/218] Update server.ts --- src/server.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/server.ts b/src/server.ts index 7fefaa3f..71c9097c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -8,6 +8,7 @@ import { setupPlugins } from './plugins' import { startWorker } from './worker' import { startFastifyInstance, stopFastifyInstance } from './web/server' import { Worker } from 'celery/worker' +import { EventsProcessor } from 'ingestion/process-event' export const defaultConfig: PluginsServerConfig = { CELERY_DEFAULT_QUEUE: 'celery', @@ -19,6 +20,8 @@ export const defaultConfig: PluginsServerConfig = { DISABLE_WEB: false, WEB_PORT: 3008, WEB_HOSTNAME: '0.0.0.0', + KAFKA_HOSTS: '', + EE_ENABLED: false, } export async function startPluginsServer(config: PluginsServerConfig): Promise { @@ -29,13 +32,14 @@ export async function startPluginsServer(config: PluginsServerConfig): Promise Date: Fri, 4 Dec 2020 11:02:05 +0100 Subject: [PATCH 021/218] Port element functions --- src/ingestion/element.ts | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/ingestion/element.ts diff --git a/src/ingestion/element.ts b/src/ingestion/element.ts new file mode 100644 index 00000000..af620f79 --- /dev/null +++ b/src/ingestion/element.ts @@ -0,0 +1,39 @@ +import { Element } from 'types' + +function _escape(input: string): string { + return input.replace(/"/g, '\\"') +} + +export function elements_to_string(elements: Element[]): string { + const ret = elements.map((element) => { + let el_string = '' + if (element.tag_name) { + el_string += element.tag_name + } + if (element.attr_class) { + element.attr_class.sort() + for (const single_class of element.attr_class) { + el_string += `.${single_class.replace(/"/g, '')}` + } + } + let attributes: Record = { + ...(element.text ? { text: element.text } : {}), + 'nth-child': element.nth_child ?? 0, + 'nth-of-type': element.nth_of_type ?? 0, + ...(element.href ? { href: element.href } : {}), + ...(element.attr_id ? { attr_id: element.attr_id } : {}), + ...element.attributes, + } + attributes = Object.fromEntries( + Object.entries(attributes) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => [_escape(key.toString()), _escape(value.toString())]) + ) + el_string += ':' + el_string += Object.entries(attributes) + .map(([key, value]) => `${key}=${value}`) + .join('') + return el_string + }) + return ret.join(';') +} From f9b88bb7bbec282e3e8762fd78851dfbe084e0d6 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Dec 2020 11:02:41 +0100 Subject: [PATCH 022/218] Use Kafka producer stream and make overall optimizations --- src/ingestion/process-event.ts | 82 +++++++++++++++++++--------------- src/server.ts | 5 ++- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 134cf6ff..528aa79c 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,16 +1,32 @@ -import { KafkaConsumer } from 'node-rdkafka' +import { KafkaConsumer, Producer, ProducerStream } from 'node-rdkafka' import { DateTime, Duration } from 'luxon' -import { UUID, UUIDT } from './utils' +import { loadSync } from 'protobufjs' import { PluginsServer, Data, Properties, Element, Team, Event, Person } from 'types' -import { KAFKA_EVENTS_WAL } from './topics' +import { UUID, UUIDT } from './utils' +import { KAFKA_EVENTS, KAFKA_EVENTS_WAL } from './topics' +import { elements_to_string } from './element' +import { join } from 'path' + +const root = loadSync(join(__dirname, 'idl/events.proto')) +const EventProto = root.lookupType('Event') export class EventsProcessor { pluginsServer: PluginsServer kafkaConsumer: KafkaConsumer + kafkaProducerStream: ProducerStream constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer this.kafkaConsumer = this.buildKafkaConsumer() + this.kafkaProducerStream = Producer.createWriteStream( + { + 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, + }, + {}, + { + topic: KAFKA_EVENTS, + } + ) } buildKafkaConsumer(): KafkaConsumer { @@ -64,7 +80,7 @@ export class EventsProcessor { team_id: number, now: DateTime, sent_at: DateTime | null - ): void { + ): Promise { const properties: Properties = data.properties ?? {} if (data['$set']) { properties['$set'] = data['$set'] @@ -205,8 +221,8 @@ export class EventsProcessor { distinct_id: string, properties: Properties, timestamp: DateTime, - sent_at: DateTime - ): void { + sent_at: DateTime | null + ): Promise { const elements: Record[] | undefined = properties['$elements'] let elements_list: Element[] = [] if (elements && elements.length) { @@ -255,7 +271,7 @@ export class EventsProcessor { rows: Person[] } = await this.pluginsServer.db.query( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6)', - [sent_at.toISO(), {}, team_id, null, false, person_uuid.toString()] + [sent_at ? sent_at.toISO() : DateTime.utc(), {}, team_id, null, false, person_uuid.toString()] ) await this.pluginsServer.db.query( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3)', @@ -264,15 +280,7 @@ export class EventsProcessor { } catch {} } - this.create_event( - event_uuid, - event, - team, - distinct_id, - properties, - timestamp, - elements_list, - ) + this.create_event(event_uuid, event, team, distinct_id, properties, timestamp, elements_list) } create_event( @@ -283,28 +291,28 @@ export class EventsProcessor { properties?: Properties, timestamp?: DateTime | string, elements?: Element[] - ): void { - timestamp ??= DateTime.utc() - + ): string { + if (!timestamp) timestamp = DateTime.utc() + // ClickHouse specific formatting timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() - - let elements_chain = elements && elements.length ? elements_to_string(elements=elements) : '' - - pb_event = events_pb2.Event() - pb_event.uuid = str(event_uuid) - pb_event.event = event - pb_event.properties = json.dumps(properties ?? {}) - pb_event.timestamp = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") - pb_event.team_id = team.pk - pb_event.distinct_id = str(distinct_id) - pb_event.elements_chain = elements_chain - pb_event.created_at = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") - - p = ClickhouseProducer() - - p.produce_proto(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=pb_event) - - return str(event_uuid) + + const elements_chain = elements && elements.length ? elements_to_string(elements) : '' + const eventUuidString = event_uuid.toString() + + const message = EventProto.create({ + uuid: eventUuidString, + event, + properties: JSON.stringify(properties ?? {}), + timestamp: timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u'), + team_id: team.id, + distinct_id, + elements_chain, + created_at: timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u'), + }) + + this.kafkaProducerStream.write(EventProto.encode(message).finish()) + + return eventUuidString } } diff --git a/src/server.ts b/src/server.ts index 71c9097c..89bcc633 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,8 +7,8 @@ import { version } from '../package.json' import { setupPlugins } from './plugins' import { startWorker } from './worker' import { startFastifyInstance, stopFastifyInstance } from './web/server' -import { Worker } from 'celery/worker' -import { EventsProcessor } from 'ingestion/process-event' +import { Worker } from './celery/worker' +import { EventsProcessor } from './ingestion/process-event' export const defaultConfig: PluginsServerConfig = { CELERY_DEFAULT_QUEUE: 'celery', @@ -39,6 +39,7 @@ export async function startPluginsServer(config: PluginsServerConfig): Promise Date: Fri, 4 Dec 2020 11:32:18 +0100 Subject: [PATCH 023/218] Add session recording --- src/ingestion/process-event.ts | 140 +++++++++++++++------------------ src/ingestion/utils.ts | 9 +++ src/server.ts | 3 +- 3 files changed, 73 insertions(+), 79 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 528aa79c..b06dce72 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,9 +1,9 @@ import { KafkaConsumer, Producer, ProducerStream } from 'node-rdkafka' -import { DateTime, Duration } from 'luxon' +import { DateTime, DateTimeOptions, Duration } from 'luxon' import { loadSync } from 'protobufjs' import { PluginsServer, Data, Properties, Element, Team, Event, Person } from 'types' -import { UUID, UUIDT } from './utils' -import { KAFKA_EVENTS, KAFKA_EVENTS_WAL } from './topics' +import { castTimestampOrNow, UUID, UUIDT } from './utils' +import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' import { join } from 'path' @@ -13,12 +13,13 @@ const EventProto = root.lookupType('Event') export class EventsProcessor { pluginsServer: PluginsServer kafkaConsumer: KafkaConsumer - kafkaProducerStream: ProducerStream + kafkaProducerStreamEvent: ProducerStream + kafkaProducerStreamSessionRecording: ProducerStream constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer this.kafkaConsumer = this.buildKafkaConsumer() - this.kafkaProducerStream = Producer.createWriteStream( + this.kafkaProducerStreamEvent = Producer.createWriteStream( { 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, }, @@ -27,6 +28,15 @@ export class EventsProcessor { topic: KAFKA_EVENTS, } ) + this.kafkaProducerStreamSessionRecording = Producer.createWriteStream( + { + 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, + }, + {}, + { + topic: KAFKA_SESSION_RECORDING_EVENTS, + } + ) } buildKafkaConsumer(): KafkaConsumer { @@ -72,6 +82,12 @@ export class EventsProcessor { this.kafkaConsumer.connect() } + stop(): void { + this.kafkaConsumer.disconnect() + this.kafkaProducerStreamEvent.destroy() + this.kafkaProducerStreamSessionRecording.destroy() + } + async process_event_ee( distinct_id: string, ip: string, @@ -89,10 +105,9 @@ export class EventsProcessor { const person_uuid = new UUIDT() const event_uuid = new UUIDT() const ts = this.handle_timestamp(data, now, sent_at) - // TODO: this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) + this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) if (data['event'] === '$snapshot') { - /* TODO: this.create_session_recording_event( event_uuid, team_id, @@ -101,7 +116,6 @@ export class EventsProcessor { ts, properties['$snapshot_data'] ) - */ } else { await this._capture_ee( event_uuid, @@ -139,78 +153,28 @@ export class EventsProcessor { return now } - /* - function handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { - if (event === "$create_alias") { + handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { + if (event === '$create_alias') { + /* TODO: _alias( properties["alias"], distinct_id, team_id, ) - } else if (event === "$identify") { - if (properties["$anon_distinct_id"]) { + */ + } else if (event === '$identify') { + if (properties['$anon_distinct_id']) { + /* TODO: _alias( properties["$anon_distinct_id"], distinct_id, team_id, ) + */ } - if (properties["$set"]) { - _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) + if (properties['$set']) { + // TODO: _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) } - _set_is_identified(team_id=team_id, distinct_id=distinct_id) + // TODO: _set_is_identified(team_id=team_id, distinct_id=distinct_id) } } - function _alias(previous_distinct_id: string, distinct_id: string, team_id: number, retry_if_failed = true): void { - let old_person: Person | null = null - let new_person: Person | null = null - - try { - old_person = Person.objects.get( - team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=previous_distinct_id - ) - except Person.DoesNotExist: - pass - - try: - new_person = Person.objects.get( - team_id=team_id, persondistinctid__team_id=team_id, persondistinctid__distinct_id=distinct_id - ) - except Person.DoesNotExist: - pass - - if old_person and not new_person: - try: - old_person.add_distinct_id(distinct_id) - # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id - except IntegrityError: - if retry_if_failed: # run everything again to merge the users if needed - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if not old_person and new_person: - try: - new_person.add_distinct_id(previous_distinct_id) - # Catch race case when somebody already added this distinct_id between .get and .add_distinct_id - except IntegrityError: - if retry_if_failed: # run everything again to merge the users if needed - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if not old_person and not new_person: - try: - Person.objects.create( - team_id=team_id, distinct_ids=[str(distinct_id), str(previous_distinct_id)], - ) - # Catch race condition where in between getting and creating, another request already created this user. - except IntegrityError: - if retry_if_failed: - # try once more, probably one of the two persons exists now - _alias(previous_distinct_id, distinct_id, team_id, False) - return - - if old_person and new_person and old_person != new_person: - new_person.merge_people([old_person]) - } - */ - async _capture_ee( event_uuid: UUID, person_uuid: UUID, @@ -292,11 +256,7 @@ export class EventsProcessor { timestamp?: DateTime | string, elements?: Element[] ): string { - if (!timestamp) timestamp = DateTime.utc() - - // ClickHouse specific formatting - timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() - + const timestampString = castTimestampOrNow(timestamp) const elements_chain = elements && elements.length ? elements_to_string(elements) : '' const eventUuidString = event_uuid.toString() @@ -304,15 +264,41 @@ export class EventsProcessor { uuid: eventUuidString, event, properties: JSON.stringify(properties ?? {}), - timestamp: timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u'), + timestamp: timestampString, team_id: team.id, distinct_id, elements_chain, - created_at: timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u'), + created_at: timestampString, }) - this.kafkaProducerStream.write(EventProto.encode(message).finish()) + this.kafkaProducerStreamEvent.write(EventProto.encode(message).finish()) return eventUuidString } + + create_session_recording_event( + uuid: UUID, + team_id: number, + distinct_id: string, + session_id: string, + timestamp: DateTime | string, + snapshot_data: Record + ): string { + const timestampString = castTimestampOrNow(timestamp) + const uuidString = uuid.toString() + + const data = { + uuid: uuidString, + team_id: team_id, + distinct_id: distinct_id, + session_id: session_id, + snapshot_data: JSON.stringify(snapshot_data), + timestamp: timestampString, + created_at: timestampString, + } + + this.kafkaProducerStreamSessionRecording.write(Buffer.from(JSON.stringify(data))) + + return uuidString + } } diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 86cb9b55..ec824744 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -145,3 +145,12 @@ export class UUIDT extends UUID { export function isLooselyFalsy(value: any): boolean { return Array.isArray(value) ? !value.length : !value || !Object.keys(value).length } + +function castTimestampOrNow(timestamp?: DateTime | string | null): string { + if (!timestamp) timestamp = DateTime.utc() + + // ClickHouse specific formatting + timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() + + return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') +} diff --git a/src/server.ts b/src/server.ts index 89bcc633..2a09ab2f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -38,8 +38,7 @@ export async function startPluginsServer(config: PluginsServerConfig): Promise Date: Fri, 4 Dec 2020 15:05:40 +0100 Subject: [PATCH 024/218] Add plugin processing --- src/ingestion/process-event.ts | 29 +++++++++++++++++------------ src/types.ts | 1 - 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index b06dce72..6196bf03 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,11 +1,13 @@ -import { KafkaConsumer, Producer, ProducerStream } from 'node-rdkafka' -import { DateTime, DateTimeOptions, Duration } from 'luxon' +import { KafkaConsumer, Message, Producer, ProducerStream } from 'node-rdkafka' +import { DateTime, Duration } from 'luxon' import { loadSync } from 'protobufjs' import { PluginsServer, Data, Properties, Element, Team, Event, Person } from 'types' import { castTimestampOrNow, UUID, UUIDT } from './utils' import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' import { join } from 'path' +import { runPlugins } from 'plugins' +import { PluginEvent } from 'posthog-plugins' const root = loadSync(join(__dirname, 'idl/events.proto')) const EventProto = root.lookupType('Event') @@ -60,16 +62,19 @@ export class EventsProcessor { }) .on('data', async (message) => { // TODO: time with statsd - const event = JSON.parse(message.value!.toString()) as Event - await this.process_event_ee( - event.distinct_id, - event.ip, - event.site_url, - event.data, - event.team_id, - DateTime.fromISO(event.now), - event.sent_at ? DateTime.fromISO(event.sent_at) : null - ) + let event: Event = JSON.parse(message.value!.toString()) + event = await runPlugins(this.pluginsServer, event) + if (event) { + await this.process_event_ee( + event.distinct_id, + event.ip, + event.site_url, + event.data, + event.team_id, + DateTime.fromISO(event.now), + event.sent_at ? DateTime.fromISO(event.sent_at) : null + ) + } kafkaConsumer.commitMessage(message) }) diff --git a/src/types.ts b/src/types.ts index e8bac29b..64abcc55 100644 --- a/src/types.ts +++ b/src/types.ts @@ -127,7 +127,6 @@ export type Data = Record export type Properties = Record export interface Event { - id: number distinct_id: string ip: string site_url: string From 11700f6d5584d278acd1ade4e7e02b6cea29c0c8 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 7 Dec 2020 13:29:09 +0100 Subject: [PATCH 025/218] Fix castTimestampOrNow --- src/ingestion/utils.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index ec824744..9f358bb9 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -146,8 +146,9 @@ export function isLooselyFalsy(value: any): boolean { return Array.isArray(value) ? !value.length : !value || !Object.keys(value).length } -function castTimestampOrNow(timestamp?: DateTime | string | null): string { - if (!timestamp) timestamp = DateTime.utc() +export function castTimestampOrNow(timestamp?: DateTime | string | null): string { + if (!timestamp) + timestamp = DateTime.utc() // ClickHouse specific formatting timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() From b18735e31c3987fe0391a1862c5fe2a430c1372e Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 7 Dec 2020 14:14:47 +0100 Subject: [PATCH 026/218] Update yarn.lock --- yarn.lock | 256 +++++++++++++++--------------------------------------- 1 file changed, 69 insertions(+), 187 deletions(-) diff --git a/yarn.lock b/yarn.lock index 7dfed766..d8d2a01e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@assemblyscript/loader@^0.10.1": + version "0.10.1" + resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.10.1.tgz#70e45678f06c72fa2e350e8553ec4a4d72b92e06" + integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== + "@babel/cli@^7.0.0": version "7.12.1" resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.12.1.tgz#e08a0b1cb6fcd4b9eb6a606ba5602c5c0fe24a0c" @@ -175,7 +180,7 @@ dependencies: "@babel/types" "^7.12.1" -"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.12.1": +"@babel/helper-module-imports@^7.12.1": version "7.12.5" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz#1bfc0229f794988f76ed0a4d4e90860850b54dfb" integrity sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA== @@ -1236,22 +1241,6 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= -"@rollup/plugin-json@^4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@rollup/plugin-json/-/plugin-json-4.1.0.tgz#54e09867ae6963c593844d8bd7a9c718294496f3" - integrity sha512-yfLbTdNS6amI/2OpmbiBoW12vngr5NW2jCJVZSBEz+H5KfUJZ2M7sDjk0U6GOOdCWFVScShte29o9NezJ53TPw== - dependencies: - "@rollup/pluginutils" "^3.0.8" - -"@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b" - integrity sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg== - dependencies: - "@types/estree" "0.0.39" - estree-walker "^1.0.1" - picomatch "^2.2.2" - "@sinonjs/commons@^1.7.0": version "1.8.1" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.1.tgz#e7df00f98a203324f6dc7cc606cad9d4a8ab2217" @@ -1311,16 +1300,6 @@ resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.2.tgz#66ad9331f63fe8a3d3d9d8c6e3906dd10f6446e8" integrity sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog== -"@types/estree@*": - version "0.0.45" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.45.tgz#e9387572998e5ecdac221950dab3e8c3b16af884" - integrity sha512-jnqIUKDUqJbDIUxm0Uj7bnlMnRm1T/eZ9N+AVMqhPgzrba2GhGG5o/jCTwmdPK709nEZsGoMzXEDUjcXHa3W0g== - -"@types/estree@0.0.39": - version "0.0.39" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f" - integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw== - "@types/graceful-fs@^4.1.2": version "4.1.4" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.4.tgz#4ff9f641a7c6d1a3508ff88bc3141b152772e753" @@ -1440,13 +1419,6 @@ resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.1.5.tgz#b6ab3bba29e16b821d84e09ecfaded462b816b00" integrity sha512-UEyp8LwZ4Dg30kVU2Q3amHHyTn1jEdhCIE59ANed76GaT1Vp76DD3ZWSAxgCrw6wJ0TqeoBpqmfUHiUDPs//HQ== -"@types/resolve@0.0.8": - version "0.0.8" - resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-0.0.8.tgz#f26074d238e02659e323ce1a13d041eee280e194" - integrity sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ== - dependencies: - "@types/node" "*" - "@types/stack-utils@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.0.tgz#7036640b4e21cc2f259ae826ce843d277dad8cff" @@ -1921,6 +1893,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= +base64-js@^1.2.0: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + base64-js@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" @@ -2050,11 +2027,6 @@ buffer@^5.5.0: base64-js "^1.3.1" ieee754 "^1.1.13" -builtin-modules@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.1.0.tgz#aad97c15131eb76b65b50ef208e7584cd76a7484" - integrity sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw== - cache-base@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2" @@ -2301,11 +2273,6 @@ commander@^6.2.0: resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.0.tgz#b990bfb8ac030aedc6d11bc04d1488ffef56db75" integrity sha512-zP4jEKbe8SHzKJYQmq8Y9gYjtO/POJLgIdKgV7B9qNmABVFVc+ctqSX6iXh4mCpJfRBOabiZ2YKPg8ciDw6C+Q== -commondir@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" - integrity sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs= - compare-versions@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.6.0.tgz#1a5689913685e5a87637b8d3ffca75514ec41d62" @@ -2983,16 +2950,6 @@ estraverse@^5.1.0, estraverse@^5.2.0: resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880" integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ== -estree-walker@^0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362" - integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w== - -estree-walker@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-1.0.1.tgz#31bc5d612c96b704106b477e6dd5d8aa138cb700" - integrity sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg== - esutils@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" @@ -3006,6 +2963,11 @@ event-emitter@~0.3.5: d "1" es5-ext "~0.10.14" +eventemitter-asyncresource@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b" + integrity sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ== + exec-sh@^0.3.2: version "0.3.4" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" @@ -3272,15 +3234,6 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -find-cache-dir@^3.3.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.3.1.tgz#89b33fad4a4670daa94f855f7fbe31d6d84fe880" - integrity sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ== - dependencies: - commondir "^1.0.1" - make-dir "^3.0.2" - pkg-dir "^4.1.0" - find-my-way@^3.0.5: version "3.0.5" resolved "https://registry.yarnpkg.com/find-my-way/-/find-my-way-3.0.5.tgz#f71c5ef1b4865401e1b97ba428121a8f55439eec" @@ -3389,15 +3342,6 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== -fs-extra@8.1.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" - integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^4.0.0" - universalify "^0.1.0" - fs-readdir-recursive@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz#e32fc030a2ccee44a6b5371308da54be0b397d27" @@ -3552,7 +3496,7 @@ globby@^11.0.1: merge2 "^1.3.0" slash "^3.0.0" -graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4: +graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.2.4: version "4.2.4" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== @@ -3628,6 +3572,20 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +hdr-histogram-js@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/hdr-histogram-js/-/hdr-histogram-js-2.0.1.tgz#ecb1ff2bcb6181c3e93ff4af9472c28c7e97284e" + integrity sha512-uPZxl1dAFnjUFHWLZmt93vUUvtHeaBay9nVNHu38SdOjMSF/4KqJUqa1Seuj08ptU1rEb6AHvB41X8n/zFZ74Q== + dependencies: + "@assemblyscript/loader" "^0.10.1" + base64-js "^1.2.0" + pako "^1.0.3" + +hdr-histogram-percentiles-obj@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/hdr-histogram-percentiles-obj/-/hdr-histogram-percentiles-obj-3.0.0.tgz#9409f4de0c2dda78e61de2d9d78b1e9f3cba283c" + integrity sha512-7kIufnBqdsBGcSZLPJwqHT3yhk1QTsSlFsVD3kx5ixH/AlgBs9yM1q6DPhXZ8f8gtdqgh7N7/5btRLpQsS2gHw== + hosted-git-info@^2.1.4: version "2.8.8" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" @@ -3957,11 +3915,6 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" -is-module@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591" - integrity sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE= - is-nan@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.3.0.tgz#85d1f5482f7051c2019f5673ccebdb06f3b0db03" @@ -4003,13 +3956,6 @@ is-potential-custom-element-name@^1.0.0: resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.0.tgz#0c52e54bcca391bb2c494b21e8626d7336c6e397" integrity sha1-DFLlS8yjkbssSUsh6GJtczbG45c= -is-reference@^1.1.2: - version "1.2.1" - resolved "https://registry.yarnpkg.com/is-reference/-/is-reference-1.2.1.tgz#8b2dac0b371f4bc994fdeaba9eb542d03002d0b7" - integrity sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ== - dependencies: - "@types/estree" "*" - is-regex@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.1.tgz#c6f98aacc546f6cec5468a07b7b153ab564a57b9" @@ -4606,13 +4552,6 @@ json5@^1.0.1: dependencies: minimist "^1.2.0" -jsonfile@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" - integrity sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss= - optionalDependencies: - graceful-fs "^4.1.6" - jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -4840,13 +4779,6 @@ luxon@^1.25.0: resolved "https://registry.yarnpkg.com/luxon/-/luxon-1.25.0.tgz#d86219e90bc0102c0eb299d65b2f5e95efe1fe72" integrity sha512-hEgLurSH8kQRjY6i4YLey+mcKVAWXbDNlZRmM6AgWDJ1cY3atl8Ztf5wEY7VBReFbmGnwQPz7KYJblL8B2k0jQ== -magic-string@^0.25.2: - version "0.25.7" - resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051" - integrity sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA== - dependencies: - sourcemap-codec "^1.4.4" - make-dir@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5" @@ -4855,7 +4787,7 @@ make-dir@^2.1.0: pify "^4.0.1" semver "^5.6.0" -make-dir@^3.0.0, make-dir@^3.0.2: +make-dir@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== @@ -5057,16 +4989,34 @@ next-tick@~1.0.0: resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" integrity sha1-yobR/ogoFpsBICCOPchCS524NCw= +nice-napi@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/nice-napi/-/nice-napi-1.0.2.tgz#dc0ab5a1eac20ce548802fc5686eaa6bc654927b" + integrity sha512-px/KnJAJZf5RuBGcfD+Sp2pAKq0ytz8j+1NehvgIGFkvtvFrDM3T8E4x/JJODXK9WZow8RRGrbA9QQ3hs+pDhA== + dependencies: + node-addon-api "^3.0.0" + node-gyp-build "^4.2.2" + nice-try@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +node-addon-api@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.0.2.tgz#04bc7b83fd845ba785bb6eae25bc857e1ef75681" + integrity sha512-+D4s2HCnxPd5PjjI0STKwncjXTUKKqm74MDMz9OPXavjsGmjkvwgLtA5yoxJUdmpj52+2u+RrXgPipahKczMKg== + node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-gyp-build@^4.2.2: + version "4.2.3" + resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739" + integrity sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg== + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -5347,6 +5297,11 @@ packet-reader@1.0.0: resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== +pako@^1.0.3: + version "1.0.11" + resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" + integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" @@ -5500,7 +5455,7 @@ pgpass@1.x: dependencies: split2 "^3.1.1" -picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1, picomatch@^2.2.2: +picomatch@^2.0.4, picomatch@^2.0.5, picomatch@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg== @@ -5551,6 +5506,17 @@ pirates@^4.0.1: dependencies: node-modules-regexp "^1.0.0" +piscina@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/piscina/-/piscina-2.1.0.tgz#947d937e0efd0de7a0e8bf10363c2e053c619413" + integrity sha512-3FgX36QyZcU4prKuNKl7/lWlOF3HAv9n7JpCjw09Zbql2KkzXXQ7E5xUS+RV5wV24Rn0r6Lr8jLdtU/cNZHAnA== + dependencies: + eventemitter-asyncresource "^1.0.0" + hdr-histogram-js "^2.0.1" + hdr-histogram-percentiles-obj "^3.0.0" + optionalDependencies: + nice-napi "^1.0.2" + pkg-dir@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b" @@ -5558,7 +5524,7 @@ pkg-dir@^2.0.0: dependencies: find-up "^2.1.0" -pkg-dir@^4.1.0, pkg-dir@^4.2.0: +pkg-dir@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== @@ -6029,13 +5995,6 @@ resolve-url@^0.2.1: resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= -resolve@1.17.0: - version "1.17.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.17.0.tgz#b25941b54968231cc2d1bb76a79cb7f2c0bf8444" - integrity sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w== - dependencies: - path-parse "^1.0.6" - resolve@^1.0.0: version "1.19.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" @@ -6044,7 +6003,7 @@ resolve@^1.0.0: is-core-module "^2.1.0" path-parse "^1.0.6" -resolve@^1.10.0, resolve@^1.10.1, resolve@^1.11.0, resolve@^1.11.1, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.3.2: +resolve@^1.10.0, resolve@^1.10.1, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.3.2: version "1.18.1" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.18.1.tgz#018fcb2c5b207d2a6424aee361c5a266da8f4130" integrity sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA== @@ -6101,68 +6060,6 @@ rimraf@^3.0.0: dependencies: glob "^7.1.3" -rollup-plugin-babel@^4.4.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-babel/-/rollup-plugin-babel-4.4.0.tgz#d15bd259466a9d1accbdb2fe2fff17c52d030acb" - integrity sha512-Lek/TYp1+7g7I+uMfJnnSJ7YWoD58ajo6Oarhlex7lvUce+RCKRuGRSgztDO3/MF/PuGKmUL5iTHKf208UNszw== - dependencies: - "@babel/helper-module-imports" "^7.0.0" - rollup-pluginutils "^2.8.1" - -rollup-plugin-commonjs@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-commonjs/-/rollup-plugin-commonjs-10.1.0.tgz#417af3b54503878e084d127adf4d1caf8beb86fb" - integrity sha512-jlXbjZSQg8EIeAAvepNwhJj++qJWNJw1Cl0YnOqKtP5Djx+fFGkp3WRh+W0ASCaFG5w1jhmzDxgu3SJuVxPF4Q== - dependencies: - estree-walker "^0.6.1" - is-reference "^1.1.2" - magic-string "^0.25.2" - resolve "^1.11.0" - rollup-pluginutils "^2.8.1" - -rollup-plugin-dts@^1.4.9: - version "1.4.13" - resolved "https://registry.yarnpkg.com/rollup-plugin-dts/-/rollup-plugin-dts-1.4.13.tgz#4f086e84f4fdcc1f49160799ebc66f6b09db292b" - integrity sha512-7mxoQ6PcmCkBE5ZhrjGDL4k42XLy8BkSqpiRi1MipwiGs+7lwi4mQkp2afX+OzzLjJp/TGM8llfe8uayIUhPEw== - optionalDependencies: - "@babel/code-frame" "^7.10.4" - -rollup-plugin-node-resolve@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-5.2.0.tgz#730f93d10ed202473b1fb54a5997a7db8c6d8523" - integrity sha512-jUlyaDXts7TW2CqQ4GaO5VJ4PwwaV8VUGA7+km3n6k6xtOEacf61u0VXwN80phY/evMcaS+9eIeJ9MOyDxt5Zw== - dependencies: - "@types/resolve" "0.0.8" - builtin-modules "^3.1.0" - is-module "^1.0.0" - resolve "^1.11.1" - rollup-pluginutils "^2.8.1" - -rollup-plugin-typescript2@^0.29.0: - version "0.29.0" - resolved "https://registry.yarnpkg.com/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.29.0.tgz#b7ad83f5241dbc5bdf1e98d9c3fca005ffe39e1a" - integrity sha512-YytahBSZCIjn/elFugEGQR5qTsVhxhUwGZIsA9TmrSsC88qroGo65O5HZP/TTArH2dm0vUmYWhKchhwi2wL9bw== - dependencies: - "@rollup/pluginutils" "^3.1.0" - find-cache-dir "^3.3.1" - fs-extra "8.1.0" - resolve "1.17.0" - tslib "2.0.1" - -rollup-pluginutils@^2.8.1: - version "2.8.2" - resolved "https://registry.yarnpkg.com/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz#72f2af0748b592364dbd3389e600e5a9444a351e" - integrity sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ== - dependencies: - estree-walker "^0.6.1" - -rollup@^2.22.1: - version "2.33.1" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.33.1.tgz#802795164164ee63cd47769d8879c33ec8ae0c40" - integrity sha512-uY4O/IoL9oNW8MMcbA5hcOaz6tZTMIh7qJHx/tzIJm+n1wLoY38BLn6fuy7DhR57oNFLMbDQtDeJoFURt5933w== - optionalDependencies: - fsevents "~2.1.2" - rsvp@^4.8.4: version "4.8.5" resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734" @@ -6457,11 +6354,6 @@ source-map@^0.7.3: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383" integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ== -sourcemap-codec@^1.4.4: - version "1.4.8" - resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz#ea804bd94857402e6992d05a38ef1ae35a9ab4c4" - integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== - spdx-correct@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" @@ -6955,11 +6847,6 @@ tsconfig@^7.0.0: strip-bom "^3.0.0" strip-json-comments "^2.0.0" -tslib@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.1.tgz#410eb0d113e5b6356490eec749603725b021b43e" - integrity sha512-SgIkNheinmEBgx1IUNirK0TUD4X9yjjBRTqqjggWCU3pUEqIk3/Uwl3yRixYKT6WjQuGiwDv4NomL3wqRCj+CQ== - tslib@^1.8.1, tslib@^1.9.0: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" @@ -7073,11 +6960,6 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" -universalify@^0.1.0: - version "0.1.2" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" - integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== - unset-value@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559" From 5ee99a58f2e233404feba0cede286be8c0fb8694 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 7 Dec 2020 14:46:19 +0100 Subject: [PATCH 027/218] Improve typing and consume event messages in batches of 100 --- src/ingestion/process-event.ts | 41 +++++++++++++++++++--------------- src/types.ts | 10 --------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 6196bf03..f90ffba8 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -54,29 +54,34 @@ export class EventsProcessor { kafkaConsumer .on('ready', () => { kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) - kafkaConsumer.consume() + // consume event messages in batches of 100 + kafkaConsumer.consume(100, async (error, messages) => { + if (error) { + throw error // TODO: handle errors in a smarter way + } + for (const message of messages) { + // TODO: time with statsd + let event: PluginEvent | null = JSON.parse(message.value!.toString()) as PluginEvent + event = await runPlugins(this.pluginsServer, event) + if (event) { + await this.process_event_ee( + event.distinct_id, + event.ip, + event.site_url, + event, + event.team_id, + DateTime.fromISO(event.now), + event.sent_at ? DateTime.fromISO(event.sent_at) : null + ) + } + } + kafkaConsumer.commit() + }) // consume in batches of 100 console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) }) .on('disconnected', () => { console.info(`๐Ÿ›‘ Kafka consumer disconnected!`) }) - .on('data', async (message) => { - // TODO: time with statsd - let event: Event = JSON.parse(message.value!.toString()) - event = await runPlugins(this.pluginsServer, event) - if (event) { - await this.process_event_ee( - event.distinct_id, - event.ip, - event.site_url, - event.data, - event.team_id, - DateTime.fromISO(event.now), - event.sent_at ? DateTime.fromISO(event.sent_at) : null - ) - } - kafkaConsumer.commitMessage(message) - }) return kafkaConsumer } diff --git a/src/types.ts b/src/types.ts index aca8c679..52a25e96 100644 --- a/src/types.ts +++ b/src/types.ts @@ -141,16 +141,6 @@ export interface Team { export type Data = Record export type Properties = Record -export interface Event { - distinct_id: string - ip: string - site_url: string - data: Data - team_id: number - now: string - sent_at: string -} - export interface Element { text: string tag_name: string From a181259e0e1bc603a9812b2cfffb59ec42efefb1 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 7 Dec 2020 16:28:54 +0100 Subject: [PATCH 028/218] Improve code clarity --- src/ingestion/process-event.ts | 50 ++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index f90ffba8..7026cacd 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,5 +1,5 @@ -import { KafkaConsumer, Message, Producer, ProducerStream } from 'node-rdkafka' -import { DateTime, Duration } from 'luxon' +import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' +import { DateTime, DateTimeOptions, Duration } from 'luxon' import { loadSync } from 'protobufjs' import { PluginsServer, Data, Properties, Element, Team, Event, Person } from 'types' import { castTimestampOrNow, UUID, UUIDT } from './utils' @@ -51,32 +51,34 @@ export class EventsProcessor { } ) + const processBatch = async (error: LibrdKafkaError, messages: Message[]): Promise => { + if (error) { + throw error // TODO: handle errors in a smarter way + } + for (const message of messages) { + // TODO: time with statsd + let event: PluginEvent | null = JSON.parse(message.value!.toString()) as PluginEvent + event = await runPlugins(this.pluginsServer, event) + if (event) { + await this.process_event_ee( + event.distinct_id, + event.ip, + event.site_url, + event, + event.team_id, + DateTime.fromISO(event.now), + event.sent_at ? DateTime.fromISO(event.sent_at) : null + ) + } + } + kafkaConsumer.commit() + } + kafkaConsumer .on('ready', () => { kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) // consume event messages in batches of 100 - kafkaConsumer.consume(100, async (error, messages) => { - if (error) { - throw error // TODO: handle errors in a smarter way - } - for (const message of messages) { - // TODO: time with statsd - let event: PluginEvent | null = JSON.parse(message.value!.toString()) as PluginEvent - event = await runPlugins(this.pluginsServer, event) - if (event) { - await this.process_event_ee( - event.distinct_id, - event.ip, - event.site_url, - event, - event.team_id, - DateTime.fromISO(event.now), - event.sent_at ? DateTime.fromISO(event.sent_at) : null - ) - } - } - kafkaConsumer.commit() - }) // consume in batches of 100 + kafkaConsumer.consume(100, processBatch) console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) }) .on('disconnected', () => { From 2e7d476fb32fb9b2e88b6d5009705172ba4a3aa2 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 7 Dec 2020 17:00:40 +0100 Subject: [PATCH 029/218] Add timing with StatsD --- package.json | 1 + src/ingestion/process-event.ts | 5 +++-- src/server.ts | 11 ++++++++++- src/types.ts | 10 +++++++--- yarn.lock | 19 +++++++++++++++++-- 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index af0d6cdf..1bc62a89 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@types/luxon": "^1.25.0", "adm-zip": "^0.4.16", "fastify": "^3.8.0", + "hot-shots": "^8.2.1", "ioredis": "^4.19.2", "luxon": "^1.25.0", "node-fetch": "^2.6.1", diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 7026cacd..87e4436b 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,7 +1,7 @@ import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' import { DateTime, DateTimeOptions, Duration } from 'luxon' import { loadSync } from 'protobufjs' -import { PluginsServer, Data, Properties, Element, Team, Event, Person } from 'types' +import { PluginsServer, Data, Properties, Element, Team, Person } from 'types' import { castTimestampOrNow, UUID, UUIDT } from './utils' import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' @@ -56,7 +56,7 @@ export class EventsProcessor { throw error // TODO: handle errors in a smarter way } for (const message of messages) { - // TODO: time with statsd + const timer = new Date() let event: PluginEvent | null = JSON.parse(message.value!.toString()) as PluginEvent event = await runPlugins(this.pluginsServer, event) if (event) { @@ -70,6 +70,7 @@ export class EventsProcessor { event.sent_at ? DateTime.fromISO(event.sent_at) : null ) } + this.pluginsServer.statsd?.timing(`${this.pluginsServer.STATSD_PREFIX}_posthog_cloud`, timer) } kafkaConsumer.commit() } diff --git a/src/server.ts b/src/server.ts index 8f3d26cd..d6eebe80 100644 --- a/src/server.ts +++ b/src/server.ts @@ -9,6 +9,7 @@ import { Worker } from 'celery/worker' import { version } from '../package.json' import { PluginEvent } from 'posthog-plugins' import Piscina from 'piscina' +import { StatsD } from 'hot-shots' function overrideWithEnv(config: PluginsServerConfig): PluginsServerConfig { const newConfig: Record = { ...config } @@ -31,6 +32,8 @@ export const defaultConfig: PluginsServerConfig = overrideWithEnv({ WEB_PORT: 3008, WEB_HOSTNAME: '0.0.0.0', WORKER_CONCURRENCY: 0, // use all cores + STATSD_PORT: 8125, + STATSD_PREFIX: '' }) export async function createServer( @@ -47,11 +50,17 @@ export async function createServer( const redis = new Redis(serverConfig.REDIS_URL) + const statsd = serverConfig.STATSD_HOST ? new StatsD({ + port: serverConfig.STATSD_PORT, + host: serverConfig.STATSD_HOST, + }) : null + + const server: PluginsServer = { ...serverConfig, db, redis, - + statsd, plugins: new Map(), pluginConfigs: new Map(), pluginConfigsPerTeam: new Map(), diff --git a/src/types.ts b/src/types.ts index 52a25e96..d0b5a35a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,13 +3,14 @@ import { Redis } from 'ioredis' import { PluginEvent, PluginAttachment, PluginConfigSchema } from 'posthog-plugins' import { VM, VMScript } from 'vm2' import { DateTime } from 'luxon' +import { StatsD } from 'hot-shots' export interface PluginsServerConfig { WORKER_CONCURRENCY: number CELERY_DEFAULT_QUEUE: string DATABASE_URL: string // Postgres database URL - KAFKA_HOSTS: string // comma-delimited Kafka hosts - EE_ENABLED: boolean // whether EE features Kafka + ClickHouse are enabled + KAFKA_HOSTS?: string // comma-delimited Kafka hosts + EE_ENABLED?: boolean // whether EE features Kafka + ClickHouse are enabled PLUGINS_CELERY_QUEUE: string REDIS_URL: string BASE_DIR: string @@ -17,6 +18,9 @@ export interface PluginsServerConfig { DISABLE_WEB: boolean WEB_PORT: number WEB_HOSTNAME: string + STATSD_HOST?: string + STATSD_PORT: number + STATSD_PREFIX: string __jestMock?: { getPluginRows: Plugin[] @@ -29,7 +33,7 @@ export interface PluginsServer extends PluginsServerConfig { // active connections to postgres and redis db: Pool redis: Redis - + statsd: StatsD | null // currently enabled plugin status plugins: Map pluginConfigs: Map diff --git a/yarn.lock b/yarn.lock index d8d2a01e..1236fd57 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1933,7 +1933,7 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== -bindings@^1.3.1, bindings@^1.5.0: +bindings@^1.3.0, bindings@^1.3.1, bindings@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== @@ -3591,6 +3591,13 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg== +hot-shots@^8.2.1: + version "8.2.1" + resolved "https://registry.yarnpkg.com/hot-shots/-/hot-shots-8.2.1.tgz#8e7845f3978d7e834db2c1b47302de7ced2258c9" + integrity sha512-yI2wOPc40MNOqF+Q/8S+WzCQJqRNDpW/7bY0oEiiAzvDKQSHscCxK31eQiBuE1mJTTKmlZ+Q9rP8zRpxnpctzw== + optionalDependencies: + unix-dgram "2.0.x" + html-encoding-sniffer@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" @@ -4957,7 +4964,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -nan@^2.12.1, nan@^2.14.0: +nan@^2.12.1, nan@^2.13.2, nan@^2.14.0: version "2.14.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== @@ -6960,6 +6967,14 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" +unix-dgram@2.0.x: + version "2.0.4" + resolved "https://registry.yarnpkg.com/unix-dgram/-/unix-dgram-2.0.4.tgz#14d4fc21e539742b8fb027de16eccd4e5503a344" + integrity sha512-7tpK6x7ls7J7pDrrAU63h93R0dVhRbPwiRRCawR10cl+2e1VOvF3bHlVJc6WI1dl/8qk5He673QU+Ogv7bPNaw== + dependencies: + bindings "^1.3.0" + nan "^2.13.2" + unset-value@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559" From 9ae0f9aa9196e34c09fa41259bfe62f26869fb10 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 7 Dec 2020 17:12:53 +0100 Subject: [PATCH 030/218] Update style --- src/ingestion/utils.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 9f358bb9..0d5819bc 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -123,7 +123,7 @@ export class UUIDT extends UUID { const array = new Uint8Array(16) // 48 bits for time, WILL FAIL in 10 895 CE // XXXXXXXX-XXXX-****-****-************ - // TODO: why the hell are the two leftmost octets (four hexadecimal chars) always zero? + // TODO: why the hell are the two leftmost octets (first four hexadecimal chars) always zero? for (let i = 5; i >= 0; i--) { array[i] = unixTimeMs & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits unixTimeMs >>>= 8 // remove these last 8 binary digits @@ -147,8 +147,9 @@ export function isLooselyFalsy(value: any): boolean { } export function castTimestampOrNow(timestamp?: DateTime | string | null): string { - if (!timestamp) + if (!timestamp) { timestamp = DateTime.utc() + } // ClickHouse specific formatting timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() From 698ddfad41fe57e7f149f8638abdf0be877924ef Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 7 Dec 2020 17:14:34 +0100 Subject: [PATCH 031/218] Format --- .devcontainer/docker-compose.yml | 2 ++ src/server.ts | 13 +++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 8b78f8c9..5b43dcee 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -24,6 +24,7 @@ services: # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db + # Uncomment the next line to use a non-root user for all processes. # user: node # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. @@ -39,6 +40,7 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward MongoDB locally. # (Adding the "ports" property to this file will not forward from a Codespace.) redis: diff --git a/src/server.ts b/src/server.ts index d6eebe80..9f52aacc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -33,7 +33,7 @@ export const defaultConfig: PluginsServerConfig = overrideWithEnv({ WEB_HOSTNAME: '0.0.0.0', WORKER_CONCURRENCY: 0, // use all cores STATSD_PORT: 8125, - STATSD_PREFIX: '' + STATSD_PREFIX: '', }) export async function createServer( @@ -50,11 +50,12 @@ export async function createServer( const redis = new Redis(serverConfig.REDIS_URL) - const statsd = serverConfig.STATSD_HOST ? new StatsD({ - port: serverConfig.STATSD_PORT, - host: serverConfig.STATSD_HOST, - }) : null - + const statsd = serverConfig.STATSD_HOST + ? new StatsD({ + port: serverConfig.STATSD_PORT, + host: serverConfig.STATSD_HOST, + }) + : null const server: PluginsServer = { ...serverConfig, From 3488d047dc68607506c2be0796eb1d8244c58dc0 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 8 Dec 2020 15:22:32 +0100 Subject: [PATCH 032/218] Merge and alias --- src/ingestion/process-event.ts | 174 ++++++++++++++++++++++++++++----- src/types.ts | 15 ++- 2 files changed, 166 insertions(+), 23 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 87e4436b..85271821 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,13 +1,14 @@ import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' import { DateTime, DateTimeOptions, Duration } from 'luxon' import { loadSync } from 'protobufjs' -import { PluginsServer, Data, Properties, Element, Team, Person } from 'types' +import { PluginsServer, Data, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' import { castTimestampOrNow, UUID, UUIDT } from './utils' import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' import { join } from 'path' import { runPlugins } from 'plugins' import { PluginEvent } from 'posthog-plugins' +import { exception } from 'console' const root = loadSync(join(__dirname, 'idl/events.proto')) const EventProto = root.lookupType('Event') @@ -168,18 +169,10 @@ export class EventsProcessor { handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { if (event === '$create_alias') { - /* TODO: - _alias( - properties["alias"], distinct_id, team_id, - ) - */ + this._alias(properties['alias'], distinct_id, team_id) } else if (event === '$identify') { if (properties['$anon_distinct_id']) { - /* TODO: - _alias( - properties["$anon_distinct_id"], distinct_id, team_id, - ) - */ + this._alias(properties['$anon_distinct_id'], distinct_id, team_id) } if (properties['$set']) { // TODO: _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) @@ -188,6 +181,123 @@ export class EventsProcessor { } } + async _alias( + previous_distinct_id: string, + distinct_id: string, + team_id: number, + retry_if_failed = true + ): Promise { + const old_person: Person | undefined = ( + await this.pluginsServer.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + [team_id, previous_distinct_id] + ) + ).rows[0] + + const new_person: Person | undefined = ( + await this.pluginsServer.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + + if (old_person && !new_person) { + try { + this.add_distinct_id(old_person, distinct_id) + // Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + } catch { + // integrity error + if (retry_if_failed) { + // run everything again to merge the users if needed + this._alias(previous_distinct_id, distinct_id, team_id, false) + } + } + return + } + + if (!old_person && new_person) { + try { + this.add_distinct_id(new_person, previous_distinct_id) + // Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + } catch { + // integrity error + if (retry_if_failed) { + // run everything again to merge the users if needed + this._alias(previous_distinct_id, distinct_id, team_id, false) + } + } + return + } + + if (!old_person && !new_person) { + try { + const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, false, new UUIDT()) + this.add_distinct_id(personCreated, distinct_id) + this.add_distinct_id(personCreated, previous_distinct_id) + // Catch race condition where in between getting and creating, another request already created this user. + } catch { + // integrity error + if (retry_if_failed) { + // try once more, probably one of the two persons exists now + this._alias(previous_distinct_id, distinct_id, team_id, false) + } + } + return + } + + if (old_person && new_person && old_person.id !== new_person.id) { + this.merge_people(new_person, [old_person]) + } + } + + async merge_people(merge_into: Person, people_to_merge: Person[]): Promise { + let first_seen = merge_into.created_at + + // merge the properties + for (const other_person of people_to_merge) { + merge_into.properties = { ...other_person.properties, ...merge_into.properties } + if (other_person.created_at < first_seen) { + // Keep the oldest created_at (i.e. the first time we've seen this person) + first_seen = other_person.created_at + } + } + + await this.pluginsServer.db.query('UPDATE posthog_person SET created_at = $1 WHERE id = $2', [ + first_seen.toISO(), + merge_into.id, + ]) + + // merge the distinct_ids + for (const other_person of people_to_merge) { + const other_person_distinct_ids: PersonDistinctId[] = ( + await this.pluginsServer.db.query( + 'SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', + [other_person, merge_into.team_id] + ) + ).rows + for (const person_distinct_id of other_person_distinct_ids) { + await this.pluginsServer.db.query('UPDATE posthog_persondistinctid SET person_id = $1 WHERE id = $2', [ + merge_into.id, + person_distinct_id.id, + ]) + } + + const other_person_cohort_ids: CohortPeople[] = ( + await this.pluginsServer.db.query('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [ + other_person.id, + ]) + ).rows + for (const person_cohort_id of other_person_cohort_ids) { + await this.pluginsServer.db.query('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ + merge_into.id, + person_cohort_id.id, + ]) + } + + await this.pluginsServer.db.query('DELETE FROM posthog_person WHERE id = $1', [other_person.id]) + } + } + async _capture_ee( event_uuid: UUID, person_uuid: UUID, @@ -242,24 +352,44 @@ export class EventsProcessor { if (!pdiCount) { // Catch race condition where in between getting and creating, another request already created this user try { - const { - rows: [personCreated], - }: { - rows: Person[] - } = await this.pluginsServer.db.query( - 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6)', - [sent_at ? sent_at.toISO() : DateTime.utc(), {}, team_id, null, false, person_uuid.toString()] - ) - await this.pluginsServer.db.query( - 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3)', - [distinct_id, personCreated.id, team_id] + const personCreated: Person = await this.create_person( + sent_at || DateTime.utc(), + {}, + team_id, + null, + false, + person_uuid.toString() ) + await this.add_distinct_id(personCreated, distinct_id) } catch {} } this.create_event(event_uuid, event, team, distinct_id, properties, timestamp, elements_list) } + async create_person( + created_at: DateTime, + properties: Properties, + team_id: number, + is_user_id: number | null, + is_identified: boolean, + uuid: UUID | string + ): Promise { + return ( + await this.pluginsServer.db.query( + 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6)', + [created_at.toISO(), properties, team_id, is_user_id, is_identified, uuid.toString()] + ) + ).rows[0] + } + + async add_distinct_id(person: Person, distinct_id: string): Promise { + await this.pluginsServer.db.query( + 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3)', + [distinct_id, person.id, person.team_id] + ) + } + create_event( event_uuid: UUID, event: string, diff --git a/src/types.ts b/src/types.ts index d0b5a35a..66a0b551 100644 --- a/src/types.ts +++ b/src/types.ts @@ -161,9 +161,22 @@ export type User = Record // not really typed as not needed so far export interface Person { id: number created_at: DateTime - team: Team + team_id: number properties: Properties is_user: User is_identified: boolean uuid: string } + +export interface PersonDistinctId { + id: number + team_id: number + person_id: number + distinct_id: string +} + +export interface CohortPeople { + id: number + cohort_id: number + person_id: number +} From 049feaf9c24a9354d00791e614451074e59b2180 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 8 Dec 2020 15:56:27 +0100 Subject: [PATCH 033/218] Are last stuff --- .devcontainer/docker-compose.yml | 2 - src/ingestion/process-event.ts | 127 +++++++++++++++++++++++++++++-- src/types.ts | 8 +- 3 files changed, 123 insertions(+), 14 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 5b43dcee..8b78f8c9 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -24,7 +24,6 @@ services: # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db - # Uncomment the next line to use a non-root user for all processes. # user: node # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. @@ -40,7 +39,6 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres - # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward MongoDB locally. # (Adding the "ports" property to this file will not forward from a Codespace.) redis: diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 85271821..fcbcd006 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,5 +1,5 @@ import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' -import { DateTime, DateTimeOptions, Duration } from 'luxon' +import { DateTime, Duration } from 'luxon' import { loadSync } from 'protobufjs' import { PluginsServer, Data, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' import { castTimestampOrNow, UUID, UUIDT } from './utils' @@ -8,7 +8,6 @@ import { elements_to_string } from './element' import { join } from 'path' import { runPlugins } from 'plugins' import { PluginEvent } from 'posthog-plugins' -import { exception } from 'console' const root = loadSync(join(__dirname, 'idl/events.proto')) const EventProto = root.lookupType('Event') @@ -167,17 +166,88 @@ export class EventsProcessor { return now } - handle_identify_or_alias(event: string, properties: Properties, distinct_id: string, team_id: number): void { + async handle_identify_or_alias( + event: string, + properties: Properties, + distinct_id: string, + team_id: number + ): Promise { if (event === '$create_alias') { - this._alias(properties['alias'], distinct_id, team_id) + await this._alias(properties['alias'], distinct_id, team_id) } else if (event === '$identify') { if (properties['$anon_distinct_id']) { - this._alias(properties['$anon_distinct_id'], distinct_id, team_id) + await this._alias(properties['$anon_distinct_id'], distinct_id, team_id) } if (properties['$set']) { - // TODO: _update_person_properties(team_id=team_id, distinct_id=distinct_id, properties=properties["$set"]) + this._update_person_properties(team_id, distinct_id, properties['$set']) + } + this._set_is_identified(team_id, distinct_id) + } + } + + async _set_is_identified(team_id: number, distinct_id: string, is_identified = true): Promise { + let personFound: Person | undefined + personFound = ( + await this.pluginsServer.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + if (!personFound) { + try { + const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, true, new UUIDT()) + this.add_distinct_id(personCreated, distinct_id) + // Catch race condition where in between getting and creating, another request already created this person + } catch { + personFound = ( + await this.pluginsServer.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + } + } + if (personFound && !personFound.is_identified) { + await this.pluginsServer.db.query('UPDATE posthog_person SET is_identified = 1 WHERE id = $1', [ + personFound.id, + ]) + } + } + + async _update_person_properties(team_id: number, distinct_id: string, properties: Properties): Promise { + let personFound: Person | undefined + personFound = ( + await this.pluginsServer.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + if (!personFound) { + try { + const personCreated = await this.create_person( + DateTime.utc(), + properties, + team_id, + null, + false, + new UUIDT() + ) + await this.add_distinct_id(personCreated, distinct_id) + // Catch race condition where in between getting and creating, another request already created this person + } catch { + personFound = ( + await this.pluginsServer.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] } - // TODO: _set_is_identified(team_id=team_id, distinct_id=distinct_id) + } + if (personFound) { + this.pluginsServer.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ + { ...personFound.properties, ...properties }, + personFound.id, + ]) } } @@ -338,7 +408,7 @@ export class EventsProcessor { properties['$ip'] = ip } - // TODO: store_names_and_properties(team=team, event=event, properties=properties) + this.store_names_and_properties(team, event, properties) const { rows: [{ pdiCount }], @@ -367,6 +437,47 @@ export class EventsProcessor { this.create_event(event_uuid, event, team, distinct_id, properties, timestamp, elements_list) } + async store_names_and_properties(team: Team, event: string, properties: Properties): Promise { + // In _capture we only prefetch a couple of fields in Team to avoid fetching too much data + let save = false + if (!team.ingested_event) { + // First event for the team captured + // TODO: capture "first team event ingested" + team.ingested_event = true + save = true + } + if (!(event in team.event_names)) { + save = true + team.event_names.push(event) + team.event_names_with_usage.push({ event: event, usage_count: null, volume: null }) + } + for (const [key, value] of Object.entries(properties)) { + if (!(key in team.event_properties)) { + team.event_properties.push(key) + team.event_properties_with_usage.push({ key: key, usage_count: null, volume: null }) + save = true + } + if (typeof value === 'number' && !(key in team.event_properties_numerical)) { + team.event_properties_numerical.push(key) + save = true + } + } + if (save) { + await this.pluginsServer.db.query( + 'UPDATE posthog_team SET ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, event_properties_with_usage = $5, event_properties_numerical = $6 WHERE id = $7', + [ + team.ingested_event, + team.event_names, + team.event_names_with_usage, + team.event_properties, + team.event_names_with_usage, + team.event_properties_numerical, + team.id, + ] + ) + } + } + async create_person( created_at: DateTime, properties: Properties, diff --git a/src/types.ts b/src/types.ts index 66a0b551..acc5c5c4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -113,14 +113,14 @@ export interface PluginConfigVMReponse { export interface EventUsage { event: string - usage_count: number - volume: number + usage_count: number | null + volume: number | null } export interface PropertyUsage { key: string - usage_count: number - volume: number + usage_count: number | null + volume: number | null } export interface Team { From f4b7f979dbdb64473c38b6e2c97440079b304990 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 05:31:21 +0100 Subject: [PATCH 034/218] Reimplement Protobuf with protos compiled to JS+TS --- package.json | 5 ++++- src/{ingestion => }/idl/events.proto | 0 .../idl/google/protobuf/timestamp.proto | 0 src/idl/protos.d.ts | 1 + src/idl/protos.js | 14 ++++++++++++++ src/ingestion/idl/element.proto | 19 ------------------- src/ingestion/idl/omni_person.proto | 12 ------------ src/ingestion/idl/person.proto | 9 --------- src/ingestion/process-event.ts | 16 +++++++--------- 9 files changed, 26 insertions(+), 50 deletions(-) rename src/{ingestion => }/idl/events.proto (100%) rename src/{ingestion => }/idl/google/protobuf/timestamp.proto (100%) create mode 100644 src/idl/protos.d.ts create mode 100644 src/idl/protos.js delete mode 100644 src/ingestion/idl/element.proto delete mode 100644 src/ingestion/idl/omni_person.proto delete mode 100644 src/ingestion/idl/person.proto diff --git a/package.json b/package.json index 1bc62a89..1e6d95fe 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,9 @@ "start:dev": "ts-node-dev --exit-child src/index.ts --config \"{\\\"BASE_DIR\\\": \\\"../posthog\\\"}\"", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", - "compile": "tsc -p .", + "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.protos && pbts -o protos.d.ts protos.js", + "compile:typescript": "tsc -p .", + "compile": "yarn compile:protobuf && yarn compile:typescript", "lint": "eslint .", "lint:fix": "eslint --fix .", "prettier": "prettier --write .", @@ -91,6 +93,7 @@ } }, "lint-staged": { + "*.{js,ts}": "eslint --fix", "*.{js,css,scss}": "prettier --write" } } diff --git a/src/ingestion/idl/events.proto b/src/idl/events.proto similarity index 100% rename from src/ingestion/idl/events.proto rename to src/idl/events.proto diff --git a/src/ingestion/idl/google/protobuf/timestamp.proto b/src/idl/google/protobuf/timestamp.proto similarity index 100% rename from src/ingestion/idl/google/protobuf/timestamp.proto rename to src/idl/google/protobuf/timestamp.proto diff --git a/src/idl/protos.d.ts b/src/idl/protos.d.ts new file mode 100644 index 00000000..10f7f3d8 --- /dev/null +++ b/src/idl/protos.d.ts @@ -0,0 +1 @@ +import * as $protobuf from 'protobufjs' diff --git a/src/idl/protos.js b/src/idl/protos.js new file mode 100644 index 00000000..a11535c7 --- /dev/null +++ b/src/idl/protos.js @@ -0,0 +1,14 @@ +/*eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars*/ +'use strict' + +var $protobuf = require('protobufjs/minimal') + +// Common aliases +var $Reader = $protobuf.Reader, + $Writer = $protobuf.Writer, + $util = $protobuf.util + +// Exported root namespace +var $root = $protobuf.roots['default'] || ($protobuf.roots['default'] = {}) + +module.exports = $root diff --git a/src/ingestion/idl/element.proto b/src/ingestion/idl/element.proto deleted file mode 100644 index a96a0c6b..00000000 --- a/src/ingestion/idl/element.proto +++ /dev/null @@ -1,19 +0,0 @@ -syntax = "proto3"; -import "google/protobuf/timestamp.proto"; - -message Element { - string uuid = 1; - string event_uuid = 2; - string text = 3; - string tag_name = 4; - string href = 5; - string attr_id = 6; - repeated string attr_class = 7; - uint64 nth_child = 8; - uint64 nth_of_type = 9; - string attributes = 10; - uint64 order = 11; - uint64 team_id = 12; - google.protobuf.Timestamp created_at = 13; - string elements_hash = 14; -} \ No newline at end of file diff --git a/src/ingestion/idl/omni_person.proto b/src/ingestion/idl/omni_person.proto deleted file mode 100644 index d327e1ac..00000000 --- a/src/ingestion/idl/omni_person.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; -import "google/protobuf/timestamp.proto"; - -message Person { - string uuid = 1; - string event_uuid = 2; - uint64 team_id = 3; - string distinct_id = 4; - string properties = 5; - bool is_identified = 6; - google.protobuf.Timestamp ts = 7; -} diff --git a/src/ingestion/idl/person.proto b/src/ingestion/idl/person.proto deleted file mode 100644 index 221f5264..00000000 --- a/src/ingestion/idl/person.proto +++ /dev/null @@ -1,9 +0,0 @@ -syntax = "proto3"; -import "google/protobuf/timestamp.proto"; - -message Person { - string uuid = 1; - google.protobuf.Timestamp created_at = 2; - string properties = 3; - bool is_identified = 4; -} diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index fcbcd006..7b05494d 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -8,9 +8,7 @@ import { elements_to_string } from './element' import { join } from 'path' import { runPlugins } from 'plugins' import { PluginEvent } from 'posthog-plugins' - -const root = loadSync(join(__dirname, 'idl/events.proto')) -const EventProto = root.lookupType('Event') +import { Event as EventProto } from '../idl/protos' export class EventsProcessor { pluginsServer: PluginsServer @@ -505,13 +503,13 @@ export class EventsProcessor { event_uuid: UUID, event: string, team: Team, - distinct_id: string, + distinctId: string, properties?: Properties, timestamp?: DateTime | string, elements?: Element[] ): string { const timestampString = castTimestampOrNow(timestamp) - const elements_chain = elements && elements.length ? elements_to_string(elements) : '' + const elementsChain = elements && elements.length ? elements_to_string(elements) : '' const eventUuidString = event_uuid.toString() const message = EventProto.create({ @@ -519,10 +517,10 @@ export class EventsProcessor { event, properties: JSON.stringify(properties ?? {}), timestamp: timestampString, - team_id: team.id, - distinct_id, - elements_chain, - created_at: timestampString, + teamId: team.id, + distinctId, + elementsChain, + createdAt: timestampString, }) this.kafkaProducerStreamEvent.write(EventProto.encode(message).finish()) From 8a8fca244552a700fef63e28868d5c2be2fd6c24 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 06:10:11 +0100 Subject: [PATCH 035/218] Fix things --- package.json | 2 +- src/__tests__/ingestion/utils.test.ts | 2 +- src/idl/protos.d.ts | 247 ++++++++ src/idl/protos.js | 773 ++++++++++++++++++++++++++ src/ingestion/process-event.ts | 4 +- src/server.ts | 3 +- src/worker/__tests__/queue.test.ts | 4 +- 7 files changed, 1027 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 1e6d95fe..3a916d87 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "start:dev": "ts-node-dev --exit-child src/index.ts --config \"{\\\"BASE_DIR\\\": \\\"../posthog\\\"}\"", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", - "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.protos && pbts -o protos.d.ts protos.js", + "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && prettier --write .", "compile:typescript": "tsc -p .", "compile": "yarn compile:protobuf && yarn compile:typescript", "lint": "eslint .", diff --git a/src/__tests__/ingestion/utils.test.ts b/src/__tests__/ingestion/utils.test.ts index 033c82b3..4d0050b9 100644 --- a/src/__tests__/ingestion/utils.test.ts +++ b/src/__tests__/ingestion/utils.test.ts @@ -111,7 +111,7 @@ describe('UUIDT', () => { it('is well-formed', () => { const uuidt = new UUIDT() const uuidtString = uuidt.toString() - // UTC timestamp matching (roughly) + // UTC timestamp matching (roughly, only comparing the beginning as the timestamp's end inevitably drifts away) expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8)) // series matching expect(uuidtString.slice(14, 18)).toEqual('0000') diff --git a/src/idl/protos.d.ts b/src/idl/protos.d.ts index 10f7f3d8..bbf095d0 100644 --- a/src/idl/protos.d.ts +++ b/src/idl/protos.d.ts @@ -1 +1,248 @@ import * as $protobuf from 'protobufjs' +/** Properties of an Event. */ +export interface IEvent { + /** Event uuid */ + uuid?: string | null + + /** Event event */ + event?: string | null + + /** Event properties */ + properties?: string | null + + /** Event timestamp */ + timestamp?: string | null + + /** Event teamId */ + teamId?: number | Long | null + + /** Event distinctId */ + distinctId?: string | null + + /** Event createdAt */ + createdAt?: string | null + + /** Event elementsChain */ + elementsChain?: string | null + + /** Event protoCreatedAt */ + protoCreatedAt?: google.protobuf.ITimestamp | null + + /** Event protoTimestamp */ + protoTimestamp?: google.protobuf.ITimestamp | null +} + +/** Represents an Event. */ +export class Event implements IEvent { + /** + * Constructs a new Event. + * @param [properties] Properties to set + */ + constructor(properties?: IEvent) + + /** Event uuid. */ + public uuid: string + + /** Event event. */ + public event: string + + /** Event properties. */ + public properties: string + + /** Event timestamp. */ + public timestamp: string + + /** Event teamId. */ + public teamId: number | Long + + /** Event distinctId. */ + public distinctId: string + + /** Event createdAt. */ + public createdAt: string + + /** Event elementsChain. */ + public elementsChain: string + + /** Event protoCreatedAt. */ + public protoCreatedAt?: google.protobuf.ITimestamp | null + + /** Event protoTimestamp. */ + public protoTimestamp?: google.protobuf.ITimestamp | null + + /** + * Creates a new Event instance using the specified properties. + * @param [properties] Properties to set + * @returns Event instance + */ + public static create(properties?: IEvent): Event + + /** + * Encodes the specified Event message. Does not implicitly {@link Event.verify|verify} messages. + * @param message Event message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: IEvent, writer?: $protobuf.Writer): $protobuf.Writer + + /** + * Encodes the specified Event message, length delimited. Does not implicitly {@link Event.verify|verify} messages. + * @param message Event message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encodeDelimited(message: IEvent, writer?: $protobuf.Writer): $protobuf.Writer + + /** + * Decodes an Event message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns Event + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: $protobuf.Reader | Uint8Array, length?: number): Event + + /** + * Decodes an Event message from the specified reader or buffer, length delimited. + * @param reader Reader or buffer to decode from + * @returns Event + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decodeDelimited(reader: $protobuf.Reader | Uint8Array): Event + + /** + * Verifies an Event message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): string | null + + /** + * Creates an Event message from a plain object. Also converts values to their respective internal types. + * @param object Plain object + * @returns Event + */ + public static fromObject(object: { [k: string]: any }): Event + + /** + * Creates a plain object from an Event message. Also converts values to other types if specified. + * @param message Event + * @param [options] Conversion options + * @returns Plain object + */ + public static toObject(message: Event, options?: $protobuf.IConversionOptions): { [k: string]: any } + + /** + * Converts this Event to JSON. + * @returns JSON object + */ + public toJSON(): { [k: string]: any } +} + +/** Namespace google. */ +export namespace google { + /** Namespace protobuf. */ + namespace protobuf { + /** Properties of a Timestamp. */ + interface ITimestamp { + /** Timestamp seconds */ + seconds?: number | Long | null + + /** Timestamp nanos */ + nanos?: number | null + } + + /** Represents a Timestamp. */ + class Timestamp implements ITimestamp { + /** + * Constructs a new Timestamp. + * @param [properties] Properties to set + */ + constructor(properties?: google.protobuf.ITimestamp) + + /** Timestamp seconds. */ + public seconds: number | Long + + /** Timestamp nanos. */ + public nanos: number + + /** + * Creates a new Timestamp instance using the specified properties. + * @param [properties] Properties to set + * @returns Timestamp instance + */ + public static create(properties?: google.protobuf.ITimestamp): google.protobuf.Timestamp + + /** + * Encodes the specified Timestamp message. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. + * @param message Timestamp message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: google.protobuf.ITimestamp, writer?: $protobuf.Writer): $protobuf.Writer + + /** + * Encodes the specified Timestamp message, length delimited. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. + * @param message Timestamp message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encodeDelimited( + message: google.protobuf.ITimestamp, + writer?: $protobuf.Writer + ): $protobuf.Writer + + /** + * Decodes a Timestamp message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns Timestamp + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: $protobuf.Reader | Uint8Array, length?: number): google.protobuf.Timestamp + + /** + * Decodes a Timestamp message from the specified reader or buffer, length delimited. + * @param reader Reader or buffer to decode from + * @returns Timestamp + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decodeDelimited(reader: $protobuf.Reader | Uint8Array): google.protobuf.Timestamp + + /** + * Verifies a Timestamp message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): string | null + + /** + * Creates a Timestamp message from a plain object. Also converts values to their respective internal types. + * @param object Plain object + * @returns Timestamp + */ + public static fromObject(object: { [k: string]: any }): google.protobuf.Timestamp + + /** + * Creates a plain object from a Timestamp message. Also converts values to other types if specified. + * @param message Timestamp + * @param [options] Conversion options + * @returns Plain object + */ + public static toObject( + message: google.protobuf.Timestamp, + options?: $protobuf.IConversionOptions + ): { [k: string]: any } + + /** + * Converts this Timestamp to JSON. + * @returns JSON object + */ + public toJSON(): { [k: string]: any } + } + } +} diff --git a/src/idl/protos.js b/src/idl/protos.js index a11535c7..7f96f73e 100644 --- a/src/idl/protos.js +++ b/src/idl/protos.js @@ -11,4 +11,777 @@ var $Reader = $protobuf.Reader, // Exported root namespace var $root = $protobuf.roots['default'] || ($protobuf.roots['default'] = {}) +$root.Event = (function () { + /** + * Properties of an Event. + * @exports IEvent + * @interface IEvent + * @property {string|null} [uuid] Event uuid + * @property {string|null} [event] Event event + * @property {string|null} [properties] Event properties + * @property {string|null} [timestamp] Event timestamp + * @property {number|Long|null} [teamId] Event teamId + * @property {string|null} [distinctId] Event distinctId + * @property {string|null} [createdAt] Event createdAt + * @property {string|null} [elementsChain] Event elementsChain + * @property {google.protobuf.ITimestamp|null} [protoCreatedAt] Event protoCreatedAt + * @property {google.protobuf.ITimestamp|null} [protoTimestamp] Event protoTimestamp + */ + + /** + * Constructs a new Event. + * @exports Event + * @classdesc Represents an Event. + * @implements IEvent + * @constructor + * @param {IEvent=} [properties] Properties to set + */ + function Event(properties) { + if (properties) { + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) { + if (properties[keys[i]] != null) { + this[keys[i]] = properties[keys[i]] + } + } + } + } + + /** + * Event uuid. + * @member {string} uuid + * @memberof Event + * @instance + */ + Event.prototype.uuid = '' + + /** + * Event event. + * @member {string} event + * @memberof Event + * @instance + */ + Event.prototype.event = '' + + /** + * Event properties. + * @member {string} properties + * @memberof Event + * @instance + */ + Event.prototype.properties = '' + + /** + * Event timestamp. + * @member {string} timestamp + * @memberof Event + * @instance + */ + Event.prototype.timestamp = '' + + /** + * Event teamId. + * @member {number|Long} teamId + * @memberof Event + * @instance + */ + Event.prototype.teamId = $util.Long ? $util.Long.fromBits(0, 0, true) : 0 + + /** + * Event distinctId. + * @member {string} distinctId + * @memberof Event + * @instance + */ + Event.prototype.distinctId = '' + + /** + * Event createdAt. + * @member {string} createdAt + * @memberof Event + * @instance + */ + Event.prototype.createdAt = '' + + /** + * Event elementsChain. + * @member {string} elementsChain + * @memberof Event + * @instance + */ + Event.prototype.elementsChain = '' + + /** + * Event protoCreatedAt. + * @member {google.protobuf.ITimestamp|null|undefined} protoCreatedAt + * @memberof Event + * @instance + */ + Event.prototype.protoCreatedAt = null + + /** + * Event protoTimestamp. + * @member {google.protobuf.ITimestamp|null|undefined} protoTimestamp + * @memberof Event + * @instance + */ + Event.prototype.protoTimestamp = null + + /** + * Creates a new Event instance using the specified properties. + * @function create + * @memberof Event + * @static + * @param {IEvent=} [properties] Properties to set + * @returns {Event} Event instance + */ + Event.create = function create(properties) { + return new Event(properties) + } + + /** + * Encodes the specified Event message. Does not implicitly {@link Event.verify|verify} messages. + * @function encode + * @memberof Event + * @static + * @param {IEvent} message Event message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Event.encode = function encode(message, writer) { + if (!writer) { + writer = $Writer.create() + } + if (message.uuid != null && Object.hasOwnProperty.call(message, 'uuid')) { + writer.uint32(/* id 1, wireType 2 =*/ 10).string(message.uuid) + } + if (message.event != null && Object.hasOwnProperty.call(message, 'event')) { + writer.uint32(/* id 2, wireType 2 =*/ 18).string(message.event) + } + if (message.properties != null && Object.hasOwnProperty.call(message, 'properties')) { + writer.uint32(/* id 3, wireType 2 =*/ 26).string(message.properties) + } + if (message.timestamp != null && Object.hasOwnProperty.call(message, 'timestamp')) { + writer.uint32(/* id 4, wireType 2 =*/ 34).string(message.timestamp) + } + if (message.teamId != null && Object.hasOwnProperty.call(message, 'teamId')) { + writer.uint32(/* id 5, wireType 0 =*/ 40).uint64(message.teamId) + } + if (message.distinctId != null && Object.hasOwnProperty.call(message, 'distinctId')) { + writer.uint32(/* id 6, wireType 2 =*/ 50).string(message.distinctId) + } + if (message.createdAt != null && Object.hasOwnProperty.call(message, 'createdAt')) { + writer.uint32(/* id 7, wireType 2 =*/ 58).string(message.createdAt) + } + if (message.elementsChain != null && Object.hasOwnProperty.call(message, 'elementsChain')) { + writer.uint32(/* id 8, wireType 2 =*/ 66).string(message.elementsChain) + } + if (message.protoCreatedAt != null && Object.hasOwnProperty.call(message, 'protoCreatedAt')) { + $root.google.protobuf.Timestamp.encode( + message.protoCreatedAt, + writer.uint32(/* id 9, wireType 2 =*/ 74).fork() + ).ldelim() + } + if (message.protoTimestamp != null && Object.hasOwnProperty.call(message, 'protoTimestamp')) { + $root.google.protobuf.Timestamp.encode( + message.protoTimestamp, + writer.uint32(/* id 10, wireType 2 =*/ 82).fork() + ).ldelim() + } + return writer + } + + /** + * Encodes the specified Event message, length delimited. Does not implicitly {@link Event.verify|verify} messages. + * @function encodeDelimited + * @memberof Event + * @static + * @param {IEvent} message Event message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Event.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim() + } + + /** + * Decodes an Event message from the specified reader or buffer. + * @function decode + * @memberof Event + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {Event} Event + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Event.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) { + reader = $Reader.create(reader) + } + var end = length === undefined ? reader.len : reader.pos + length, + message = new $root.Event() + while (reader.pos < end) { + var tag = reader.uint32() + switch (tag >>> 3) { + case 1: + message.uuid = reader.string() + break + case 2: + message.event = reader.string() + break + case 3: + message.properties = reader.string() + break + case 4: + message.timestamp = reader.string() + break + case 5: + message.teamId = reader.uint64() + break + case 6: + message.distinctId = reader.string() + break + case 7: + message.createdAt = reader.string() + break + case 8: + message.elementsChain = reader.string() + break + case 9: + message.protoCreatedAt = $root.google.protobuf.Timestamp.decode(reader, reader.uint32()) + break + case 10: + message.protoTimestamp = $root.google.protobuf.Timestamp.decode(reader, reader.uint32()) + break + default: + reader.skipType(tag & 7) + break + } + } + return message + } + + /** + * Decodes an Event message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof Event + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {Event} Event + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Event.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) { + reader = new $Reader(reader) + } + return this.decode(reader, reader.uint32()) + } + + /** + * Verifies an Event message. + * @function verify + * @memberof Event + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + Event.verify = function verify(message) { + if (typeof message !== 'object' || message === null) { + return 'object expected' + } + if (message.uuid != null && message.hasOwnProperty('uuid')) { + if (!$util.isString(message.uuid)) { + return 'uuid: string expected' + } + } + if (message.event != null && message.hasOwnProperty('event')) { + if (!$util.isString(message.event)) { + return 'event: string expected' + } + } + if (message.properties != null && message.hasOwnProperty('properties')) { + if (!$util.isString(message.properties)) { + return 'properties: string expected' + } + } + if (message.timestamp != null && message.hasOwnProperty('timestamp')) { + if (!$util.isString(message.timestamp)) { + return 'timestamp: string expected' + } + } + if (message.teamId != null && message.hasOwnProperty('teamId')) { + if ( + !$util.isInteger(message.teamId) && + !(message.teamId && $util.isInteger(message.teamId.low) && $util.isInteger(message.teamId.high)) + ) { + return 'teamId: integer|Long expected' + } + } + if (message.distinctId != null && message.hasOwnProperty('distinctId')) { + if (!$util.isString(message.distinctId)) { + return 'distinctId: string expected' + } + } + if (message.createdAt != null && message.hasOwnProperty('createdAt')) { + if (!$util.isString(message.createdAt)) { + return 'createdAt: string expected' + } + } + if (message.elementsChain != null && message.hasOwnProperty('elementsChain')) { + if (!$util.isString(message.elementsChain)) { + return 'elementsChain: string expected' + } + } + if (message.protoCreatedAt != null && message.hasOwnProperty('protoCreatedAt')) { + var error = $root.google.protobuf.Timestamp.verify(message.protoCreatedAt) + if (error) { + return 'protoCreatedAt.' + error + } + } + if (message.protoTimestamp != null && message.hasOwnProperty('protoTimestamp')) { + var error = $root.google.protobuf.Timestamp.verify(message.protoTimestamp) + if (error) { + return 'protoTimestamp.' + error + } + } + return null + } + + /** + * Creates an Event message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof Event + * @static + * @param {Object.} object Plain object + * @returns {Event} Event + */ + Event.fromObject = function fromObject(object) { + if (object instanceof $root.Event) { + return object + } + var message = new $root.Event() + if (object.uuid != null) { + message.uuid = String(object.uuid) + } + if (object.event != null) { + message.event = String(object.event) + } + if (object.properties != null) { + message.properties = String(object.properties) + } + if (object.timestamp != null) { + message.timestamp = String(object.timestamp) + } + if (object.teamId != null) { + if ($util.Long) { + ;(message.teamId = $util.Long.fromValue(object.teamId)).unsigned = true + } else if (typeof object.teamId === 'string') { + message.teamId = parseInt(object.teamId, 10) + } else if (typeof object.teamId === 'number') { + message.teamId = object.teamId + } else if (typeof object.teamId === 'object') { + message.teamId = new $util.LongBits(object.teamId.low >>> 0, object.teamId.high >>> 0).toNumber(true) + } + } + if (object.distinctId != null) { + message.distinctId = String(object.distinctId) + } + if (object.createdAt != null) { + message.createdAt = String(object.createdAt) + } + if (object.elementsChain != null) { + message.elementsChain = String(object.elementsChain) + } + if (object.protoCreatedAt != null) { + if (typeof object.protoCreatedAt !== 'object') { + throw TypeError('.Event.protoCreatedAt: object expected') + } + message.protoCreatedAt = $root.google.protobuf.Timestamp.fromObject(object.protoCreatedAt) + } + if (object.protoTimestamp != null) { + if (typeof object.protoTimestamp !== 'object') { + throw TypeError('.Event.protoTimestamp: object expected') + } + message.protoTimestamp = $root.google.protobuf.Timestamp.fromObject(object.protoTimestamp) + } + return message + } + + /** + * Creates a plain object from an Event message. Also converts values to other types if specified. + * @function toObject + * @memberof Event + * @static + * @param {Event} message Event + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + Event.toObject = function toObject(message, options) { + if (!options) { + options = {} + } + var object = {} + if (options.defaults) { + object.uuid = '' + object.event = '' + object.properties = '' + object.timestamp = '' + if ($util.Long) { + var long = new $util.Long(0, 0, true) + object.teamId = + options.longs === String ? long.toString() : options.longs === Number ? long.toNumber() : long + } else { + object.teamId = options.longs === String ? '0' : 0 + } + object.distinctId = '' + object.createdAt = '' + object.elementsChain = '' + object.protoCreatedAt = null + object.protoTimestamp = null + } + if (message.uuid != null && message.hasOwnProperty('uuid')) { + object.uuid = message.uuid + } + if (message.event != null && message.hasOwnProperty('event')) { + object.event = message.event + } + if (message.properties != null && message.hasOwnProperty('properties')) { + object.properties = message.properties + } + if (message.timestamp != null && message.hasOwnProperty('timestamp')) { + object.timestamp = message.timestamp + } + if (message.teamId != null && message.hasOwnProperty('teamId')) { + if (typeof message.teamId === 'number') { + object.teamId = options.longs === String ? String(message.teamId) : message.teamId + } else { + object.teamId = + options.longs === String + ? $util.Long.prototype.toString.call(message.teamId) + : options.longs === Number + ? new $util.LongBits(message.teamId.low >>> 0, message.teamId.high >>> 0).toNumber(true) + : message.teamId + } + } + if (message.distinctId != null && message.hasOwnProperty('distinctId')) { + object.distinctId = message.distinctId + } + if (message.createdAt != null && message.hasOwnProperty('createdAt')) { + object.createdAt = message.createdAt + } + if (message.elementsChain != null && message.hasOwnProperty('elementsChain')) { + object.elementsChain = message.elementsChain + } + if (message.protoCreatedAt != null && message.hasOwnProperty('protoCreatedAt')) { + object.protoCreatedAt = $root.google.protobuf.Timestamp.toObject(message.protoCreatedAt, options) + } + if (message.protoTimestamp != null && message.hasOwnProperty('protoTimestamp')) { + object.protoTimestamp = $root.google.protobuf.Timestamp.toObject(message.protoTimestamp, options) + } + return object + } + + /** + * Converts this Event to JSON. + * @function toJSON + * @memberof Event + * @instance + * @returns {Object.} JSON object + */ + Event.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions) + } + + return Event +})() + +$root.google = (function () { + /** + * Namespace google. + * @exports google + * @namespace + */ + var google = {} + + google.protobuf = (function () { + /** + * Namespace protobuf. + * @memberof google + * @namespace + */ + var protobuf = {} + + protobuf.Timestamp = (function () { + /** + * Properties of a Timestamp. + * @memberof google.protobuf + * @interface ITimestamp + * @property {number|Long|null} [seconds] Timestamp seconds + * @property {number|null} [nanos] Timestamp nanos + */ + + /** + * Constructs a new Timestamp. + * @memberof google.protobuf + * @classdesc Represents a Timestamp. + * @implements ITimestamp + * @constructor + * @param {google.protobuf.ITimestamp=} [properties] Properties to set + */ + function Timestamp(properties) { + if (properties) { + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) { + if (properties[keys[i]] != null) { + this[keys[i]] = properties[keys[i]] + } + } + } + } + + /** + * Timestamp seconds. + * @member {number|Long} seconds + * @memberof google.protobuf.Timestamp + * @instance + */ + Timestamp.prototype.seconds = $util.Long ? $util.Long.fromBits(0, 0, false) : 0 + + /** + * Timestamp nanos. + * @member {number} nanos + * @memberof google.protobuf.Timestamp + * @instance + */ + Timestamp.prototype.nanos = 0 + + /** + * Creates a new Timestamp instance using the specified properties. + * @function create + * @memberof google.protobuf.Timestamp + * @static + * @param {google.protobuf.ITimestamp=} [properties] Properties to set + * @returns {google.protobuf.Timestamp} Timestamp instance + */ + Timestamp.create = function create(properties) { + return new Timestamp(properties) + } + + /** + * Encodes the specified Timestamp message. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. + * @function encode + * @memberof google.protobuf.Timestamp + * @static + * @param {google.protobuf.ITimestamp} message Timestamp message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Timestamp.encode = function encode(message, writer) { + if (!writer) { + writer = $Writer.create() + } + if (message.seconds != null && Object.hasOwnProperty.call(message, 'seconds')) { + writer.uint32(/* id 1, wireType 0 =*/ 8).int64(message.seconds) + } + if (message.nanos != null && Object.hasOwnProperty.call(message, 'nanos')) { + writer.uint32(/* id 2, wireType 0 =*/ 16).int32(message.nanos) + } + return writer + } + + /** + * Encodes the specified Timestamp message, length delimited. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. + * @function encodeDelimited + * @memberof google.protobuf.Timestamp + * @static + * @param {google.protobuf.ITimestamp} message Timestamp message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + Timestamp.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim() + } + + /** + * Decodes a Timestamp message from the specified reader or buffer. + * @function decode + * @memberof google.protobuf.Timestamp + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {google.protobuf.Timestamp} Timestamp + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Timestamp.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) { + reader = $Reader.create(reader) + } + var end = length === undefined ? reader.len : reader.pos + length, + message = new $root.google.protobuf.Timestamp() + while (reader.pos < end) { + var tag = reader.uint32() + switch (tag >>> 3) { + case 1: + message.seconds = reader.int64() + break + case 2: + message.nanos = reader.int32() + break + default: + reader.skipType(tag & 7) + break + } + } + return message + } + + /** + * Decodes a Timestamp message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof google.protobuf.Timestamp + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {google.protobuf.Timestamp} Timestamp + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + Timestamp.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) { + reader = new $Reader(reader) + } + return this.decode(reader, reader.uint32()) + } + + /** + * Verifies a Timestamp message. + * @function verify + * @memberof google.protobuf.Timestamp + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + Timestamp.verify = function verify(message) { + if (typeof message !== 'object' || message === null) { + return 'object expected' + } + if (message.seconds != null && message.hasOwnProperty('seconds')) { + if ( + !$util.isInteger(message.seconds) && + !( + message.seconds && + $util.isInteger(message.seconds.low) && + $util.isInteger(message.seconds.high) + ) + ) { + return 'seconds: integer|Long expected' + } + } + if (message.nanos != null && message.hasOwnProperty('nanos')) { + if (!$util.isInteger(message.nanos)) { + return 'nanos: integer expected' + } + } + return null + } + + /** + * Creates a Timestamp message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof google.protobuf.Timestamp + * @static + * @param {Object.} object Plain object + * @returns {google.protobuf.Timestamp} Timestamp + */ + Timestamp.fromObject = function fromObject(object) { + if (object instanceof $root.google.protobuf.Timestamp) { + return object + } + var message = new $root.google.protobuf.Timestamp() + if (object.seconds != null) { + if ($util.Long) { + ;(message.seconds = $util.Long.fromValue(object.seconds)).unsigned = false + } else if (typeof object.seconds === 'string') { + message.seconds = parseInt(object.seconds, 10) + } else if (typeof object.seconds === 'number') { + message.seconds = object.seconds + } else if (typeof object.seconds === 'object') { + message.seconds = new $util.LongBits( + object.seconds.low >>> 0, + object.seconds.high >>> 0 + ).toNumber() + } + } + if (object.nanos != null) { + message.nanos = object.nanos | 0 + } + return message + } + + /** + * Creates a plain object from a Timestamp message. Also converts values to other types if specified. + * @function toObject + * @memberof google.protobuf.Timestamp + * @static + * @param {google.protobuf.Timestamp} message Timestamp + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + Timestamp.toObject = function toObject(message, options) { + if (!options) { + options = {} + } + var object = {} + if (options.defaults) { + if ($util.Long) { + var long = new $util.Long(0, 0, false) + object.seconds = + options.longs === String + ? long.toString() + : options.longs === Number + ? long.toNumber() + : long + } else { + object.seconds = options.longs === String ? '0' : 0 + } + object.nanos = 0 + } + if (message.seconds != null && message.hasOwnProperty('seconds')) { + if (typeof message.seconds === 'number') { + object.seconds = options.longs === String ? String(message.seconds) : message.seconds + } else { + object.seconds = + options.longs === String + ? $util.Long.prototype.toString.call(message.seconds) + : options.longs === Number + ? new $util.LongBits(message.seconds.low >>> 0, message.seconds.high >>> 0).toNumber() + : message.seconds + } + } + if (message.nanos != null && message.hasOwnProperty('nanos')) { + object.nanos = message.nanos + } + return object + } + + /** + * Converts this Timestamp to JSON. + * @function toJSON + * @memberof google.protobuf.Timestamp + * @instance + * @returns {Object.} JSON object + */ + Timestamp.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions) + } + + return Timestamp + })() + + return protobuf + })() + + return google +})() + module.exports = $root diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 7b05494d..b648cb07 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,12 +1,10 @@ import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' import { DateTime, Duration } from 'luxon' -import { loadSync } from 'protobufjs' import { PluginsServer, Data, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' import { castTimestampOrNow, UUID, UUIDT } from './utils' import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' -import { join } from 'path' -import { runPlugins } from 'plugins' +import { runPlugins } from '../plugins' import { PluginEvent } from 'posthog-plugins' import { Event as EventProto } from '../idl/protos' diff --git a/src/server.ts b/src/server.ts index 9f52aacc..797ec85e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -10,6 +10,7 @@ import { version } from '../package.json' import { PluginEvent } from 'posthog-plugins' import Piscina from 'piscina' import { StatsD } from 'hot-shots' +import { EventsProcessor } from './ingestion/process-event' function overrideWithEnv(config: PluginsServerConfig): PluginsServerConfig { const newConfig: Record = { ...config } @@ -139,7 +140,7 @@ export async function startPluginsServer( pubSub = new Redis(server.REDIS_URL) pubSub.subscribe(server.PLUGINS_RELOAD_PUBSUB_CHANNEL) - pubSub.on('message', async (channel, message) => { + pubSub.on('message', async (channel: string, message) => { if (channel === server!.PLUGINS_RELOAD_PUBSUB_CHANNEL) { console.log('โšก Reloading plugins!') await queue?.stop() diff --git a/src/worker/__tests__/queue.test.ts b/src/worker/__tests__/queue.test.ts index 59f5b3d5..2d22fbef 100644 --- a/src/worker/__tests__/queue.test.ts +++ b/src/worker/__tests__/queue.test.ts @@ -53,7 +53,7 @@ test('worker and task passing via redis', async () => { expect(item['headers']['task']).toBe('posthog.tasks.process_event.process_event_with_plugins') expect(item['properties']['body_encoding']).toBe('base64') - const body = new Buffer(item['body'], 'base64').toString() + const body = Buffer.from(item['body'], 'base64').toString() const [args2, kwargs2] = JSON.parse(body) expect(args2).toEqual(args) @@ -74,7 +74,7 @@ test('worker and task passing via redis', async () => { expect(processedItem['headers']['task']).toBe('posthog.tasks.process_event.process_event') expect(processedItem['properties']['body_encoding']).toBe('base64') - const processedBody = new Buffer(processedItem['body'], 'base64').toString() + const processedBody = Buffer.from(processedItem['body'], 'base64').toString() const [args3, kwargs3] = JSON.parse(processedBody) expect(args3).toEqual([]) From dfd1d4612f8d082ad619d15cff9b47d1024bdb9b Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 06:23:20 +0100 Subject: [PATCH 036/218] Fix UUID#toString implementation --- src/ingestion/utils.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 0d5819bc..a9a058d0 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -103,7 +103,8 @@ export class UUIDT extends UUID { static currentSeriesPerMs: Map = new Map() /** Get per-millisecond series integer in range [0-65536). */ - static getSeries(unixTimeMs: number): number { + static getSeries(unixTimeMs: number | bigint): number { + unixTimeMs = Number(unixTimeMs) const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) if (UUIDT.currentSeriesPerMs.size > 10_000) { // Clear class dict periodically @@ -114,19 +115,19 @@ export class UUIDT extends UUID { return nextSeries } - constructor(unixTimeMs?: number) { + constructor(unixTimeMs?: number | bigint) { if (!unixTimeMs) { unixTimeMs = DateTime.utc().toMillis() } + unixTimeMs = BigInt(unixTimeMs) let series = UUIDT.getSeries(unixTimeMs) // 64 bits (8 bytes) total const array = new Uint8Array(16) // 48 bits for time, WILL FAIL in 10 895 CE // XXXXXXXX-XXXX-****-****-************ - // TODO: why the hell are the two leftmost octets (first four hexadecimal chars) always zero? for (let i = 5; i >= 0; i--) { - array[i] = unixTimeMs & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits - unixTimeMs >>>= 8 // remove these last 8 binary digits + array[i] = Number(unixTimeMs & 0xffn) // use last 8 binary digits to set UUID 2 hexadecimal digits + unixTimeMs >>= 8n // remove these last 8 binary digits } // 16 bits for series // ********-****-XXXX-****-************ From 447fb3b94aa12af8f7d9c1bc7652c89c814714c5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 09:04:56 +0100 Subject: [PATCH 037/218] Update UUID number handling --- src/__tests__/ingestion/utils.test.ts | 4 ++-- src/ingestion/utils.ts | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/__tests__/ingestion/utils.test.ts b/src/__tests__/ingestion/utils.test.ts index 4d0050b9..19628aa7 100644 --- a/src/__tests__/ingestion/utils.test.ts +++ b/src/__tests__/ingestion/utils.test.ts @@ -88,10 +88,10 @@ describe('UUID', () => { }) }) - describe('#toNumber', () => { + describe('#valueOf', () => { it('returns the right big integer', () => { const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') - expect(uuid.toNumber()).toStrictEqual(0x99abcdef123443210000dcba87654321n) + expect(uuid.valueOf()).toStrictEqual(0x99abcdef123443210000dcba87654321n) }) }) diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index a9a058d0..5c26850e 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -36,7 +36,7 @@ export class UUID { } /** Convert to 128-bit BigInt. */ - toNumber(): BigInt { + valueOf(): BigInt { let value = 0n for (const byte of this.array) { value <<= 8n @@ -103,8 +103,7 @@ export class UUIDT extends UUID { static currentSeriesPerMs: Map = new Map() /** Get per-millisecond series integer in range [0-65536). */ - static getSeries(unixTimeMs: number | bigint): number { - unixTimeMs = Number(unixTimeMs) + static getSeries(unixTimeMs: number): number { const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) if (UUIDT.currentSeriesPerMs.size > 10_000) { // Clear class dict periodically @@ -115,19 +114,19 @@ export class UUIDT extends UUID { return nextSeries } - constructor(unixTimeMs?: number | bigint) { + constructor(unixTimeMs?: number) { if (!unixTimeMs) { unixTimeMs = DateTime.utc().toMillis() } - unixTimeMs = BigInt(unixTimeMs) + let unixTimeMsBig = BigInt(unixTimeMs) let series = UUIDT.getSeries(unixTimeMs) // 64 bits (8 bytes) total const array = new Uint8Array(16) // 48 bits for time, WILL FAIL in 10 895 CE // XXXXXXXX-XXXX-****-****-************ for (let i = 5; i >= 0; i--) { - array[i] = Number(unixTimeMs & 0xffn) // use last 8 binary digits to set UUID 2 hexadecimal digits - unixTimeMs >>= 8n // remove these last 8 binary digits + array[i] = Number(unixTimeMsBig & 0xffn) // use last 8 binary digits to set UUID 2 hexadecimal digits + unixTimeMsBig >>= 8n // remove these last 8 binary digits } // 16 bits for series // ********-****-XXXX-****-************ From a79b2bfe73836ca3e1ea83dd71cd3eb3123dc359 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 09:40:08 +0100 Subject: [PATCH 038/218] Unify --- src/config.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/config.ts b/src/config.ts index ee66b571..b31e7539 100644 --- a/src/config.ts +++ b/src/config.ts @@ -17,6 +17,8 @@ export function getDefaultConfig(): PluginsServerConfig { WORKER_CONCURRENCY: 0, // use all cores TASKS_PER_WORKER: 100, LOG_LEVEL: LogLevel.Info, + STATSD_PORT: 8125, + STATSD_PREFIX: '', } } @@ -34,6 +36,11 @@ export function getConfigHelp(): Record { WORKER_CONCURRENCY: 'number of concurrent worker threads', TASKS_PER_WORKER: 'number of parallel tasks per worker thread', LOG_LEVEL: 'minimum log level', + STATSD_HOST: 'StatsD host - integration disabled if not provided', + STATSD_PORT: 'StatsD port', + STATSD_PREFIX: 'StatsD prefix', + KAFKA_HOSTS: 'comma-delimited Kafka hosts', + EE_ENABLED: 'whether EE features Kafka + ClickHouse are enabled', } } From 125f5e4a125100e85bef65ad7a165f2a4cfc58bb Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 09:40:13 +0100 Subject: [PATCH 039/218] Prettier --- .devcontainer/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 8b78f8c9..5b43dcee 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -24,6 +24,7 @@ services: # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db + # Uncomment the next line to use a non-root user for all processes. # user: node # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. @@ -39,6 +40,7 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward MongoDB locally. # (Adding the "ports" property to this file will not forward from a Codespace.) redis: From 096584483f5e09b28379207cf0f76a0e9475f018 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 09:40:19 +0100 Subject: [PATCH 040/218] Unify utils --- src/ingestion/process-event.ts | 2 +- src/ingestion/utils.ts | 158 --------------------------------- src/utils.ts | 153 +++++++++++++++++++++++++++++++ tests/ingestion/utils.test.ts | 119 ------------------------- tests/utils.test.ts | 119 +++++++++++++++++++++++++ 5 files changed, 273 insertions(+), 278 deletions(-) delete mode 100644 src/ingestion/utils.ts delete mode 100644 tests/ingestion/utils.test.ts diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index b648cb07..fe689efd 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,7 +1,7 @@ import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' import { DateTime, Duration } from 'luxon' import { PluginsServer, Data, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' -import { castTimestampOrNow, UUID, UUIDT } from './utils' +import { castTimestampOrNow, UUID, UUIDT } from '../utils' import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' import { runPlugins } from '../plugins' diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts deleted file mode 100644 index 5c26850e..00000000 --- a/src/ingestion/utils.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { randomBytes } from 'crypto' -import { DateTime } from 'luxon' - -/** LUT of byte value to hexadecimal representation. For UUID stringification. */ -const byteToHex: string[] = [] -for (let i = 0; i < 256; i++) { - byteToHex.push((i + 0x100).toString(16).substr(1)) -} - -export class UUID { - static validateString(candidate: string): void { - if (!candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i)) { - throw new Error( - 'String does not match format XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX (where each X is a hexadecimal character)!' - ) - } - } - - array: Uint8Array - - constructor(candidate: string | Uint8Array | Buffer) { - if (candidate instanceof Uint8Array) { - if (candidate.byteLength !== 16) { - throw new Error(`UUID must be built from exactly 16 bytes, but you provided ${candidate.byteLength}!`) - } - this.array = new Uint8Array(candidate) - } else { - candidate = candidate.trim() - UUID.validateString(candidate) - this.array = new Uint8Array(16) - const characters = Array.from(candidate).filter((character) => character !== '-') - for (let i = 0; i < characters.length; i += 2) { - this.array[i / 2] = parseInt(characters[i] + characters[i + 1], 16) - } - } - } - - /** Convert to 128-bit BigInt. */ - valueOf(): BigInt { - let value = 0n - for (const byte of this.array) { - value <<= 8n - value += BigInt(byte) - } - return value - } - - /** - * Convert to string format of the form: - * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX - * Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js - */ - toString(): string { - // Note: Be careful editing this code! It's been tuned for performance and works in ways you may not expect. - // See https://github.com/uuidjs/uuid/pull/434 - const arr = this.array - return ( - byteToHex[arr[0]] + - byteToHex[arr[1]] + - byteToHex[arr[2]] + - byteToHex[arr[3]] + - '-' + - byteToHex[arr[4]] + - byteToHex[arr[5]] + - '-' + - byteToHex[arr[6]] + - byteToHex[arr[7]] + - '-' + - byteToHex[arr[8]] + - byteToHex[arr[9]] + - '-' + - byteToHex[arr[10]] + - byteToHex[arr[11]] + - byteToHex[arr[12]] + - byteToHex[arr[13]] + - byteToHex[arr[14]] + - byteToHex[arr[15]] - ).toLowerCase() - } -} - -/** - * UUID (mostly) sortable by generation time. - * - * This doesn't adhere to any official UUID version spec, but it is superior as a primary key: - * to incremented integers (as they can reveal sensitive business information about usage volumes and patterns), - * to UUID v4 (as the complete randomness of v4 makes its indexing performance suboptimal), - * and to UUID v1 (as despite being time-based it can't be used practically for sorting by generation time). - * - * Order can be messed up if system Unix time is changed or if more than 65ย 536 IDs are generated per millisecond - * (that's over 5 trillion events per day), but it should be largely safe to assume that these are time-sortable. - * - * Anatomy: - * - 6 bytes - Unix time milliseconds unsigned integer - * - 2 bytes - autoincremented series unsigned integer (per millisecond, rolls over to 0 after reaching 65 535 UUIDs in one ms) - * - 8 bytes - securely random gibberish - * - * Loosely based on [Segment's KSUID](https://github.com/segmentio/ksuid) and - * on [Twitter's snowflake ID](https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html). - * Ported from the PostHog Django app. - */ -export class UUIDT extends UUID { - static currentSeriesPerMs: Map = new Map() - - /** Get per-millisecond series integer in range [0-65536). */ - static getSeries(unixTimeMs: number): number { - const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) - if (UUIDT.currentSeriesPerMs.size > 10_000) { - // Clear class dict periodically - UUIDT.currentSeriesPerMs.clear() - } - const nextSeries = typeof series === 'number' ? (series + 1) % 65_536 : 0 - UUIDT.currentSeriesPerMs.set(unixTimeMs, nextSeries) - return nextSeries - } - - constructor(unixTimeMs?: number) { - if (!unixTimeMs) { - unixTimeMs = DateTime.utc().toMillis() - } - let unixTimeMsBig = BigInt(unixTimeMs) - let series = UUIDT.getSeries(unixTimeMs) - // 64 bits (8 bytes) total - const array = new Uint8Array(16) - // 48 bits for time, WILL FAIL in 10 895 CE - // XXXXXXXX-XXXX-****-****-************ - for (let i = 5; i >= 0; i--) { - array[i] = Number(unixTimeMsBig & 0xffn) // use last 8 binary digits to set UUID 2 hexadecimal digits - unixTimeMsBig >>= 8n // remove these last 8 binary digits - } - // 16 bits for series - // ********-****-XXXX-****-************ - for (let i = 7; i >= 6; i--) { - array[i] = series & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits - series >>>= 8 // remove these last 8 binary digits - } - // 64 bits for random gibberish - // ********-****-****-XXXX-XXXXXXXXXXXX - array.set(randomBytes(8), 8) - super(array) - } -} - -/** Simplistic Python `not` equivalent. */ -export function isLooselyFalsy(value: any): boolean { - return Array.isArray(value) ? !value.length : !value || !Object.keys(value).length -} - -export function castTimestampOrNow(timestamp?: DateTime | string | null): string { - if (!timestamp) { - timestamp = DateTime.utc() - } - - // ClickHouse specific formatting - timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() - - return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') -} diff --git a/src/utils.ts b/src/utils.ts index 2ccc446c..1824cc02 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -3,6 +3,8 @@ import * as tar from 'tar-stream' import AdmZip from 'adm-zip' import * as zlib from 'zlib' import { LogLevel } from './types' +import { randomBytes } from 'crypto' +import { DateTime } from 'luxon' /** * @param binary Buffer @@ -124,3 +126,154 @@ export function cloneObject(obj: T): T { } return clone as T } + +/** LUT of byte value to hexadecimal representation. For UUID stringification. */ +const byteToHex: string[] = [] +for (let i = 0; i < 256; i++) { + byteToHex.push((i + 0x100).toString(16).substr(1)) +} + +export class UUID { + static validateString(candidate: string): void { + if (!candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i)) { + throw new Error( + 'String does not match format XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX (where each X is a hexadecimal character)!' + ) + } + } + + array: Uint8Array + + constructor(candidate: string | Uint8Array | Buffer) { + if (candidate instanceof Uint8Array) { + if (candidate.byteLength !== 16) { + throw new Error(`UUID must be built from exactly 16 bytes, but you provided ${candidate.byteLength}!`) + } + this.array = new Uint8Array(candidate) + } else { + candidate = candidate.trim() + UUID.validateString(candidate) + this.array = new Uint8Array(16) + const characters = Array.from(candidate).filter((character) => character !== '-') + for (let i = 0; i < characters.length; i += 2) { + this.array[i / 2] = parseInt(characters[i] + characters[i + 1], 16) + } + } + } + + /** Convert to 128-bit BigInt. */ + valueOf(): BigInt { + let value = 0n + for (const byte of this.array) { + value <<= 8n + value += BigInt(byte) + } + return value + } + + /** + * Convert to string format of the form: + * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + * Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js + */ + toString(): string { + // Note: Be careful editing this code! It's been tuned for performance and works in ways you may not expect. + // See https://github.com/uuidjs/uuid/pull/434 + const arr = this.array + return ( + byteToHex[arr[0]] + + byteToHex[arr[1]] + + byteToHex[arr[2]] + + byteToHex[arr[3]] + + '-' + + byteToHex[arr[4]] + + byteToHex[arr[5]] + + '-' + + byteToHex[arr[6]] + + byteToHex[arr[7]] + + '-' + + byteToHex[arr[8]] + + byteToHex[arr[9]] + + '-' + + byteToHex[arr[10]] + + byteToHex[arr[11]] + + byteToHex[arr[12]] + + byteToHex[arr[13]] + + byteToHex[arr[14]] + + byteToHex[arr[15]] + ).toLowerCase() + } +} + +/** + * UUID (mostly) sortable by generation time. + * + * This doesn't adhere to any official UUID version spec, but it is superior as a primary key: + * to incremented integers (as they can reveal sensitive business information about usage volumes and patterns), + * to UUID v4 (as the complete randomness of v4 makes its indexing performance suboptimal), + * and to UUID v1 (as despite being time-based it can't be used practically for sorting by generation time). + * + * Order can be messed up if system Unix time is changed or if more than 65ย 536 IDs are generated per millisecond + * (that's over 5 trillion events per day), but it should be largely safe to assume that these are time-sortable. + * + * Anatomy: + * - 6 bytes - Unix time milliseconds unsigned integer + * - 2 bytes - autoincremented series unsigned integer (per millisecond, rolls over to 0 after reaching 65 535 UUIDs in one ms) + * - 8 bytes - securely random gibberish + * + * Loosely based on [Segment's KSUID](https://github.com/segmentio/ksuid) and + * on [Twitter's snowflake ID](https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html). + * Ported from the PostHog Django app. + */ +export class UUIDT extends UUID { + static currentSeriesPerMs: Map = new Map() + + /** Get per-millisecond series integer in range [0-65536). */ + static getSeries(unixTimeMs: number): number { + const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) + if (UUIDT.currentSeriesPerMs.size > 10_000) { + // Clear class dict periodically + UUIDT.currentSeriesPerMs.clear() + } + const nextSeries = typeof series === 'number' ? (series + 1) % 65_536 : 0 + UUIDT.currentSeriesPerMs.set(unixTimeMs, nextSeries) + return nextSeries + } + + constructor(unixTimeMs?: number) { + if (!unixTimeMs) { + unixTimeMs = DateTime.utc().toMillis() + } + let unixTimeMsBig = BigInt(unixTimeMs) + let series = UUIDT.getSeries(unixTimeMs) + // 64 bits (8 bytes) total + const array = new Uint8Array(16) + // 48 bits for time, WILL FAIL in 10 895 CE + // XXXXXXXX-XXXX-****-****-************ + for (let i = 5; i >= 0; i--) { + array[i] = Number(unixTimeMsBig & 0xffn) // use last 8 binary digits to set UUID 2 hexadecimal digits + unixTimeMsBig >>= 8n // remove these last 8 binary digits + } + // 16 bits for series + // ********-****-XXXX-****-************ + for (let i = 7; i >= 6; i--) { + array[i] = series & 0xff // use last 8 binary digits to set UUID 2 hexadecimal digits + series >>>= 8 // remove these last 8 binary digits + } + // 64 bits for random gibberish + // ********-****-****-XXXX-XXXXXXXXXXXX + array.set(randomBytes(8), 8) + super(array) + } +} + +export function castTimestampOrNow(timestamp?: DateTime | string | null): string { + if (!timestamp) { + timestamp = DateTime.utc() + } + + // ClickHouse specific formatting + timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() + + return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') +} diff --git a/tests/ingestion/utils.test.ts b/tests/ingestion/utils.test.ts deleted file mode 100644 index 19628aa7..00000000 --- a/tests/ingestion/utils.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { randomBytes } from 'crypto' -import { UUID, UUIDT } from '../../ingestion/utils' - -describe('UUID', () => { - describe('#constructor', () => { - it('works with a valid string', () => { - const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') - expect(uuid.array).toStrictEqual( - new Uint8Array([ - 0x99, - 0xab, - 0xcd, - 0xef, - 0x12, - 0x34, - 0x43, - 0x21, - 0, - 0, - 0xdc, - 0xba, - 0x87, - 0x65, - 0x43, - 0x21, - ]) - ) - }) - - it('throws on an invalid string', () => { - expect(() => new UUID('99aBcDeF-1234-4321-WXyz-dcba87654321')).toThrow() // "WXyz" are not hexadecimal - expect(() => new UUID('99aBcDeF123443210000dcba87654321')).toThrow() // lack of hyphens - expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba87654321A')).toThrow() // one character too many - expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba8765432')).toThrow() // one character too few - expect(() => new UUID('')).toThrow() // empty string - }) - - it('works with a Uint8Array', () => { - for (let i = 0; i < 10; i++) { - const uuid = new UUID( - new Uint8Array([ - 0x99, - 0xab, - 0xcd, - 0xef, - 0x12, - 0x34, - 0x43, - 0x21, - 0, - 0, - 0xdc, - 0xba, - 0x87, - 0x65, - 0x43, - 0x21, - ]) - ) - expect(uuid.array).toStrictEqual( - new Uint8Array([ - 0x99, - 0xab, - 0xcd, - 0xef, - 0x12, - 0x34, - 0x43, - 0x21, - 0, - 0, - 0xdc, - 0xba, - 0x87, - 0x65, - 0x43, - 0x21, - ]) - ) - } - }) - - it('works with a random buffer', () => { - for (let i = 0; i < 10; i++) { - const uuid = new UUID(randomBytes(16)) - expect(uuid.array).toHaveLength(16) - } - }) - }) - - describe('#valueOf', () => { - it('returns the right big integer', () => { - const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') - expect(uuid.valueOf()).toStrictEqual(0x99abcdef123443210000dcba87654321n) - }) - }) - - describe('#toString', () => { - it('returns the right string', () => { - const original = '99aBcDeF-1234-4321-0000-dcba87654321' - const uuid = new UUID(original) - const uuidString = uuid.toString() - // 32 hexadecimal digits + 4 hyphens - expect(uuidString).toHaveLength(36) - expect(uuidString).toStrictEqual(original.toLowerCase()) - }) - }) -}) - -describe('UUIDT', () => { - it('is well-formed', () => { - const uuidt = new UUIDT() - const uuidtString = uuidt.toString() - // UTC timestamp matching (roughly, only comparing the beginning as the timestamp's end inevitably drifts away) - expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8)) - // series matching - expect(uuidtString.slice(14, 18)).toEqual('0000') - }) -}) diff --git a/tests/utils.test.ts b/tests/utils.test.ts index b55bfc94..6e89e176 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -5,7 +5,10 @@ import { bufferToStream, setLogLevel, cloneObject, + UUID, + UUIDT, } from '../src/utils' +import { randomBytes } from 'crypto' import { LogLevel } from '../src/types' // .zip in Base64: github repo posthog/helloworldplugin @@ -175,3 +178,119 @@ test('cloneObject', async () => { expect(cloneObject(o5)).toEqual(o5) expect(cloneObject(o5) === o5).toBe(true) }) +describe('UUID', () => { + describe('#constructor', () => { + it('works with a valid string', () => { + const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') + expect(uuid.array).toStrictEqual( + new Uint8Array([ + 0x99, + 0xab, + 0xcd, + 0xef, + 0x12, + 0x34, + 0x43, + 0x21, + 0, + 0, + 0xdc, + 0xba, + 0x87, + 0x65, + 0x43, + 0x21, + ]) + ) + }) + + it('throws on an invalid string', () => { + expect(() => new UUID('99aBcDeF-1234-4321-WXyz-dcba87654321')).toThrow() // "WXyz" are not hexadecimal + expect(() => new UUID('99aBcDeF123443210000dcba87654321')).toThrow() // lack of hyphens + expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba87654321A')).toThrow() // one character too many + expect(() => new UUID('99aBcDeF-1234-4321-0000-dcba8765432')).toThrow() // one character too few + expect(() => new UUID('')).toThrow() // empty string + }) + + it('works with a Uint8Array', () => { + for (let i = 0; i < 10; i++) { + const uuid = new UUID( + new Uint8Array([ + 0x99, + 0xab, + 0xcd, + 0xef, + 0x12, + 0x34, + 0x43, + 0x21, + 0, + 0, + 0xdc, + 0xba, + 0x87, + 0x65, + 0x43, + 0x21, + ]) + ) + expect(uuid.array).toStrictEqual( + new Uint8Array([ + 0x99, + 0xab, + 0xcd, + 0xef, + 0x12, + 0x34, + 0x43, + 0x21, + 0, + 0, + 0xdc, + 0xba, + 0x87, + 0x65, + 0x43, + 0x21, + ]) + ) + } + }) + + it('works with a random buffer', () => { + for (let i = 0; i < 10; i++) { + const uuid = new UUID(randomBytes(16)) + expect(uuid.array).toHaveLength(16) + } + }) + }) + + describe('#valueOf', () => { + it('returns the right big integer', () => { + const uuid = new UUID('99aBcDeF-1234-4321-0000-dcba87654321') + expect(uuid.valueOf()).toStrictEqual(0x99abcdef123443210000dcba87654321n) + }) + }) + + describe('#toString', () => { + it('returns the right string', () => { + const original = '99aBcDeF-1234-4321-0000-dcba87654321' + const uuid = new UUID(original) + const uuidString = uuid.toString() + // 32 hexadecimal digits + 4 hyphens + expect(uuidString).toHaveLength(36) + expect(uuidString).toStrictEqual(original.toLowerCase()) + }) + }) +}) + +describe('UUIDT', () => { + it('is well-formed', () => { + const uuidt = new UUIDT() + const uuidtString = uuidt.toString() + // UTC timestamp matching (roughly, only comparing the beginning as the timestamp's end inevitably drifts away) + expect(uuidtString.slice(0, 8)).toEqual(Date.now().toString(16).padStart(12, '0').slice(0, 8)) + // series matching + expect(uuidtString.slice(14, 18)).toEqual('0000') + }) +}) From 11e95e23e3a9367c8514bfed48c5265fbbb5625f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 10:21:58 +0100 Subject: [PATCH 041/218] Add newline --- tests/utils.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 6e89e176..f9fc5011 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -178,6 +178,7 @@ test('cloneObject', async () => { expect(cloneObject(o5)).toEqual(o5) expect(cloneObject(o5) === o5).toBe(true) }) + describe('UUID', () => { describe('#constructor', () => { it('works with a valid string', () => { From 7a8f82aff47b7e77c2774d74917ee252e811838b Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 10:26:05 +0100 Subject: [PATCH 042/218] Prettier but correctly --- .devcontainer/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 5b43dcee..8b78f8c9 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -24,7 +24,6 @@ services: # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db - # Uncomment the next line to use a non-root user for all processes. # user: node # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. @@ -40,7 +39,6 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres - # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward MongoDB locally. # (Adding the "ports" property to this file will not forward from a Codespace.) redis: From cdaf498ab2d72b1d65a4e472bdbc835fc5bc899f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 11:37:10 +0100 Subject: [PATCH 043/218] Fix types --- src/config.ts | 11 +++++------ tests/worker.test.ts | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/config.ts b/src/config.ts index ea0d477f..5bb763cb 100644 --- a/src/config.ts +++ b/src/config.ts @@ -40,7 +40,7 @@ export function getConfigHelp(): Record { STATSD_PORT: 'StatsD port', STATSD_PREFIX: 'StatsD prefix', KAFKA_HOSTS: 'comma-delimited Kafka hosts', - EE_ENABLED: 'whether EE features Kafka + ClickHouse are enabled', + EE_ENABLED: 'whether Enterprise Edition backend (Kafka + ClickHouse) is enabled', } } @@ -49,18 +49,17 @@ export function overrideWithEnv( env: Record = process.env ): PluginsServerConfig { const defaultConfig = getDefaultConfig() - - const newConfig: Record = { ...config } - for (const key of Object.keys(config) as PluginsServerConfigKey[]) { + const newConfig: Record = { ...config } + for (const key of Object.keys(defaultConfig) as PluginsServerConfigKey[]) { if (typeof env[key] !== 'undefined') { if (typeof defaultConfig[key] === 'number') { newConfig[key] = env[key]?.indexOf('.') ? parseFloat(env[key]!) : parseInt(env[key]!) } else if (typeof defaultConfig[key] === 'boolean') { - newConfig[key] = env[key] === 'true' || env[key] === 'True' || env[key] === '1' + newConfig[key] = env[key]?.toLowerCase() === 'true' || env[key] === '1' } else { newConfig[key] = env[key] } } } - return newConfig + return newConfig as PluginsServerConfig } diff --git a/tests/worker.test.ts b/tests/worker.test.ts index ee9c0612..05003c26 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -4,7 +4,7 @@ import { PluginEvent } from 'posthog-plugins/src/types' import { performance } from 'perf_hooks' import { mockJestWithIndex } from './helpers/plugins' import * as os from 'os' -import { LogLevel } from '../src/types' +import { LogLevel, PluginsServerConfig } from '../src/types' jest.mock('../src/sql') jest.setTimeout(300000) // 300 sec timeout @@ -58,7 +58,7 @@ function setupPiscina(workers: number, code: string, tasksPerWorker: number) { TASKS_PER_WORKER: tasksPerWorker, LOG_LEVEL: LogLevel.Log, __jestMock: mockJestWithIndex(code), - }) + } as PluginsServerConfig) } test('piscina worker test', async () => { From 9e4745b81230bd67695e816cb1c55188c49941b5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 13:05:19 +0100 Subject: [PATCH 044/218] Improve types --- src/types.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/types.ts b/src/types.ts index da00a3ab..89ff4a47 100644 --- a/src/types.ts +++ b/src/types.ts @@ -136,6 +136,15 @@ export interface PropertyUsage { volume: number | null } +export type Data = { + event: string + properties: Properties + timestamp?: string + $set?: Properties + offset?: number +} +export type Properties = Record + export interface Team { id: number name: string @@ -155,9 +164,6 @@ export interface Team { ingested_event: boolean } -export type Data = Record -export type Properties = Record - export interface Element { text: string tag_name: string @@ -169,7 +175,7 @@ export interface Element { attributes: Record } -export type User = Record // not really typed as not needed so far +export type User = Record // not really typed as not needed so far, only a placeholder for Person.is_user export interface Person { id: number From 3da3092145300090e001a51858c8c2656ecd0530 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 14:14:41 +0100 Subject: [PATCH 045/218] Bump version to 0.5.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5a576ba9..3bcf481a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "posthog-plugin-server", - "version": "0.3.4", + "version": "0.5.0", "description": "PostHog Plugin Server", "types": "dist/src/index.d.ts", "main": "dist/src/index.js", From 24a6fc8cc047c6c3347de58e01e8836ac3355d80 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 14:18:45 +0100 Subject: [PATCH 046/218] Update worker.test.ts --- tests/worker.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/worker.test.ts b/tests/worker.test.ts index 1df0b5a0..7d0adcc0 100644 --- a/tests/worker.test.ts +++ b/tests/worker.test.ts @@ -4,7 +4,7 @@ import { PluginEvent } from 'posthog-plugins/src/types' import { performance } from 'perf_hooks' import { mockJestWithIndex } from './helpers/plugins' import * as os from 'os' -import { LogLevel, PluginsServerConfig } from '../src/types' +import { LogLevel } from '../src/types' jest.mock('../src/sql') jest.setTimeout(600000) // 600 sec timeout @@ -68,7 +68,7 @@ function setupPiscina(workers: number, code: string, tasksPerWorker: number) { TASKS_PER_WORKER: tasksPerWorker, LOG_LEVEL: LogLevel.Log, __jestMock: mockJestWithIndex(code), - } as PluginsServerConfig) + }) } test('piscina worker test', async () => { From 7b50eef1642ba7de0efaa7679ea68c1b03b86325 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 14:28:14 +0100 Subject: [PATCH 047/218] Update types.ts --- src/types.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/types.ts b/src/types.ts index 94acde0f..45aeb9c3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -19,8 +19,8 @@ export interface PluginsServerConfig extends Record { TASKS_PER_WORKER: number CELERY_DEFAULT_QUEUE: string DATABASE_URL: string - KAFKA_HOSTS?: string - EE_ENABLED?: boolean + KAFKA_HOSTS: string | null + EE_ENABLED: boolean PLUGINS_CELERY_QUEUE: string REDIS_URL: string BASE_DIR: string @@ -28,7 +28,7 @@ export interface PluginsServerConfig extends Record { DISABLE_WEB: boolean WEB_PORT: number WEB_HOSTNAME: string - STATSD_HOST?: string + STATSD_HOST: string | null STATSD_PORT: number STATSD_PREFIX: string LOG_LEVEL: LogLevel From cc3de53e0fbedf08d0eb84e9b5681f728bd4f8ec Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 14:28:18 +0100 Subject: [PATCH 048/218] Update yarn.lock --- yarn.lock | 106 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 97 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index 2d393277..181424b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1241,6 +1241,74 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= +"@sentry/core@5.29.0": + version "5.29.0" + resolved "https://registry.yarnpkg.com/@sentry/core/-/core-5.29.0.tgz#4410ca0dc5785abf3df02fa23c18e83ad90d7cda" + integrity sha512-a1sZBJ2u3NG0YDlGvOTwUCWiNjhfmDtAQiKK1o6RIIbcrWy9TlSps7CYDkBP239Y3A4pnvohjEEKEP3v3L3LZQ== + dependencies: + "@sentry/hub" "5.29.0" + "@sentry/minimal" "5.29.0" + "@sentry/types" "5.29.0" + "@sentry/utils" "5.29.0" + tslib "^1.9.3" + +"@sentry/hub@5.29.0": + version "5.29.0" + resolved "https://registry.yarnpkg.com/@sentry/hub/-/hub-5.29.0.tgz#d018b978fdffc6c8261744b0d08e8d25a3f4dc58" + integrity sha512-kcDPQsRG4cFdmqDh+TzjeO7lWYxU8s1dZYAbbl1J4uGKmhNB0J7I4ak4SGwTsXLY6fhbierxr6PRaoNojCxjPw== + dependencies: + "@sentry/types" "5.29.0" + "@sentry/utils" "5.29.0" + tslib "^1.9.3" + +"@sentry/minimal@5.29.0": + version "5.29.0" + resolved "https://registry.yarnpkg.com/@sentry/minimal/-/minimal-5.29.0.tgz#bd8b52f388abcec2234dbbc6d6721ff65aa30e35" + integrity sha512-nhXofdjtO41/caiF1wk1oT3p/QuhOZDYdF/b29DoD2MiAMK9IjhhOXI/gqaRpDKkXlDvd95fDTcx4t/MqqcKXA== + dependencies: + "@sentry/hub" "5.29.0" + "@sentry/types" "5.29.0" + tslib "^1.9.3" + +"@sentry/node@^5.29.0": + version "5.29.0" + resolved "https://registry.yarnpkg.com/@sentry/node/-/node-5.29.0.tgz#409b5d8b8dc2be25c0c4ed20aadcf8a3bbe454bc" + integrity sha512-Jp32FsfkFSGVf81Hr26rGlgIwTg7Nx07mQ7rrnNuVasu6vD2aWBzUnohkkZDJ4gZRGjmk0MthukjX0RivDKcVQ== + dependencies: + "@sentry/core" "5.29.0" + "@sentry/hub" "5.29.0" + "@sentry/tracing" "5.29.0" + "@sentry/types" "5.29.0" + "@sentry/utils" "5.29.0" + cookie "^0.4.1" + https-proxy-agent "^5.0.0" + lru_map "^0.3.3" + tslib "^1.9.3" + +"@sentry/tracing@5.29.0", "@sentry/tracing@^5.29.0": + version "5.29.0" + resolved "https://registry.yarnpkg.com/@sentry/tracing/-/tracing-5.29.0.tgz#8ed515b3f9d409137357c38c8622858f9e684e4a" + integrity sha512-2ZITUH7Eur7IkmRAd5gw8Xt2Sfc28btCnT7o2P2J8ZPD65e99ATqjxXPokx0+6zEkTsstIDD3mbyuwkpbuvuTA== + dependencies: + "@sentry/hub" "5.29.0" + "@sentry/minimal" "5.29.0" + "@sentry/types" "5.29.0" + "@sentry/utils" "5.29.0" + tslib "^1.9.3" + +"@sentry/types@5.29.0": + version "5.29.0" + resolved "https://registry.yarnpkg.com/@sentry/types/-/types-5.29.0.tgz#af5cec98cde54316c14df3121f0e8106e56b578e" + integrity sha512-iDkxT/9sT3UF+Xb+JyLjZ5caMXsgLfRyV9VXQEiR2J6mgpMielj184d9jeF3bm/VMuAf/VFFqrHlcVsVgmrrMw== + +"@sentry/utils@5.29.0": + version "5.29.0" + resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-5.29.0.tgz#b4c1223ba362a94cf4850e9ca2cb24655b006b53" + integrity sha512-b2B1gshw2u3EHlAi84PuI5sfmLKXW1z9enMMhNuuNT/CoRp+g5kMAcUv/qYTws7UNnYSvTuVGuZG30v1e0hP9A== + dependencies: + "@sentry/types" "5.29.0" + tslib "^1.9.3" + "@sinonjs/commons@^1.7.0": version "1.8.1" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.1.tgz#e7df00f98a203324f6dc7cc606cad9d4a8ab2217" @@ -1588,6 +1656,13 @@ adm-zip@^0.4.16: resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.4.16.tgz#cf4c508fdffab02c269cbc7f471a875f05570365" integrity sha512-TFi4HBKSGfIKsK5YCkKaaFG2m4PEDyViZmEwof3MTIgzimHLto6muaHVpbrljdIvIrFZzEq/p4nafOeLcYegrg== +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + aggregate-error@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" @@ -2300,7 +2375,7 @@ convert-source-map@^1.1.0, convert-source-map@^1.4.0, convert-source-map@^1.6.0, dependencies: safe-buffer "~5.1.1" -cookie@^0.4.0: +cookie@^0.4.0, cookie@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1" integrity sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA== @@ -2423,6 +2498,13 @@ dateformat@~1.0.4-1.2.3: get-stdin "^4.0.1" meow "^3.3.0" +debug@4, debug@^4.0.0: + version "4.3.1" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" + integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== + dependencies: + ms "2.1.2" + debug@^2.2.0, debug@^2.3.3, debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -2430,13 +2512,6 @@ debug@^2.2.0, debug@^2.3.3, debug@^2.6.9: dependencies: ms "2.0.0" -debug@^4.0.0: - version "4.3.1" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" - integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== - dependencies: - ms "2.1.2" - debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1" @@ -3619,6 +3694,14 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" +https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" @@ -4781,6 +4864,11 @@ loud-rejection@^1.0.0: currently-unhandled "^0.4.1" signal-exit "^3.0.0" +lru_map@^0.3.3: + version "0.3.3" + resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd" + integrity sha1-tcg1G5Rky9dQM1p5ZQoOwOVhGN0= + luxon@^1.25.0: version "1.25.0" resolved "https://registry.yarnpkg.com/luxon/-/luxon-1.25.0.tgz#d86219e90bc0102c0eb299d65b2f5e95efe1fe72" @@ -6854,7 +6942,7 @@ tsconfig@^7.0.0: strip-bom "^3.0.0" strip-json-comments "^2.0.0" -tslib@^1.8.1, tslib@^1.9.0: +tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== From 182ea31f84f9cfaf54ab44f2bfb1a0536c63c19d Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 14:40:26 +0100 Subject: [PATCH 049/218] Update compile:protobuf with eslint --fix --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fbae899a..8fcab652 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "start:dev": "ts-node-dev --exit-child src/index.ts --config \"{\\\"BASE_DIR\\\": \\\"../posthog\\\"}\"", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", - "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && prettier --write .", + "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && eslint --fix . && prettier --write .", "compile:typescript": "tsc -p .", "compile": "yarn compile:protobuf && yarn compile:typescript", "lint": "eslint .", From c881fed1d16c44aa7694955969aba6e035c137ae Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 14:51:59 +0100 Subject: [PATCH 050/218] Fix typing --- src/ingestion/process-event.ts | 13 ++++++++----- src/types.ts | 6 +++--- tsconfig.json | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index fe689efd..c71d84a8 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,6 +1,6 @@ import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' import { DateTime, Duration } from 'luxon' -import { PluginsServer, Data, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' +import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' import { castTimestampOrNow, UUID, UUIDT } from '../utils' import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' @@ -15,6 +15,9 @@ export class EventsProcessor { kafkaProducerStreamSessionRecording: ProducerStream constructor(pluginsServer: PluginsServer) { + if (!pluginsServer.KAFKA_HOSTS) { + throw new Error('You must set KAFKA_HOSTS to process events from Kafka!') + } this.pluginsServer = pluginsServer this.kafkaConsumer = this.buildKafkaConsumer() this.kafkaProducerStreamEvent = Producer.createWriteStream( @@ -40,7 +43,7 @@ export class EventsProcessor { buildKafkaConsumer(): KafkaConsumer { const kafkaConsumer = new KafkaConsumer( { - 'metadata.broker.list': this.pluginsServer.KAFKA_HOSTS, + 'metadata.broker.list': this.pluginsServer.KAFKA_HOSTS!, }, { 'auto.offset.reset': 'earliest', @@ -101,7 +104,7 @@ export class EventsProcessor { distinct_id: string, ip: string, site_url: string, - data: Data, + data: EventData, team_id: number, now: DateTime, sent_at: DateTime | null @@ -141,7 +144,7 @@ export class EventsProcessor { } } - handle_timestamp(data: Data, now: DateTime, sent_at: DateTime | null): DateTime { + handle_timestamp(data: EventData, now: DateTime, sent_at: DateTime | null): DateTime { if (data['timestamp']) { if (sent_at) { // sent_at - timestamp == now - x @@ -156,7 +159,7 @@ export class EventsProcessor { } return DateTime.fromISO(data['timestamp']) } - if (!data['offset']) { + if (data['offset']) { return now.minus(Duration.fromMillis(data['offset'])) } return now diff --git a/src/types.ts b/src/types.ts index 45aeb9c3..71021ec2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -136,13 +136,13 @@ export interface PropertyUsage { volume: number | null } -export type Data = { - event: string - properties: Properties +export interface EventData extends PluginEvent { + properties?: Properties timestamp?: string $set?: Properties offset?: number } + export type Properties = Record export interface Team { diff --git a/tsconfig.json b/tsconfig.json index c606b655..4c7ac88e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,7 +12,7 @@ "sourceMap": true, "baseUrl": "./src", "outDir": "./dist", - "types": ["node", "jest"], + "types": ["node", "jest", "long"], "resolveJsonModule": true, "strict": true }, From 5da5bcb22f8d3c965b1b8be626c2942bd92913c4 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 15:00:44 +0100 Subject: [PATCH 051/218] Don't bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8fcab652..f303e504 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "posthog-plugin-server", - "version": "0.5.0", + "version": "0.4.1", "description": "PostHog Plugin Server", "types": "dist/src/index.d.ts", "main": "dist/src/index.js", From cae8ec8e757878693ba625be6d5092bfb16dfc4b Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 17:28:34 +0100 Subject: [PATCH 052/218] Fix style --- src/ingestion/process-event.ts | 2 +- src/worker/piscina.js | 2 +- src/worker/worker.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index c71d84a8..6269023b 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -56,7 +56,7 @@ export class EventsProcessor { } for (const message of messages) { const timer = new Date() - let event: PluginEvent | null = JSON.parse(message.value!.toString()) as PluginEvent + let event: EventData | null = JSON.parse(message.value!.toString()) as EventData event = await runPlugins(this.pluginsServer, event) if (event) { await this.process_event_ee( diff --git a/src/worker/piscina.js b/src/worker/piscina.js index 0c465cb6..3ece3773 100644 --- a/src/worker/piscina.js +++ b/src/worker/piscina.js @@ -10,7 +10,7 @@ if (isMainThread) { const piscina = new Piscina(createConfig(serverConfig, __filename)) piscina.on('error', (error) => { Sentry.captureException(error) - console.error('๐Ÿ”ด Piscina Worker Error!') + console.error('โš ๏ธ Piscina worker error!') console.error(error) }) return piscina diff --git a/src/worker/worker.ts b/src/worker/worker.ts index 48c817a1..1bea8101 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -7,7 +7,7 @@ import { initApp } from '../init' type TaskWorker = ({ task, args }: { task: string; args: any }) => Promise export async function createWorker(config: PluginsServerConfig): Promise { - console.info('๐Ÿงต Starting Piscina Worker Thread') + console.info('๐Ÿงต Starting Piscina worker threadโ€ฆ') initApp(config) From 81b8605375a65190960a3c2696690aa0fac7eb90 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 20:23:55 +0100 Subject: [PATCH 053/218] Use @posthog/node-rdkafka for Worker thread support --- .gitignore | 6 ++++-- package.json | 2 +- src/ingestion/process-event.ts | 3 +-- yarn.lock | 16 ++++++++-------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index b14e7ed0..8aa1a5b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .idea .clinic -node_modules +node_modules/ yarn-error.log -dist +dist/ +.yalc/ +yalc.lock diff --git a/package.json b/package.json index f303e504..c8a4de70 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "repository": "https://github.com/PostHog/posthog-plugin-server", "license": "MIT", "dependencies": { + "@posthog/node-rdkafka": "2.10.0-6.831-worker-context", "@sentry/node": "^5.29.0", "@sentry/tracing": "^5.29.0", "@types/luxon": "^1.25.0", @@ -37,7 +38,6 @@ "ioredis": "^4.19.2", "luxon": "^1.25.0", "node-fetch": "^2.6.1", - "node-rdkafka": "^2.9.1", "node-schedule": "^1.3.2", "pg": "^8.4.2", "piscina": "^2.1.0", diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 6269023b..1b21e112 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,11 +1,10 @@ -import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from 'node-rdkafka' +import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from '@posthog/node-rdkafka' import { DateTime, Duration } from 'luxon' import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' import { castTimestampOrNow, UUID, UUIDT } from '../utils' import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' import { runPlugins } from '../plugins' -import { PluginEvent } from 'posthog-plugins' import { Event as EventProto } from '../idl/protos' export class EventsProcessor { diff --git a/yarn.lock b/yarn.lock index 181424b4..cb99c2d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1188,6 +1188,14 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" +"@posthog/node-rdkafka@2.10.0-6.831-worker-context": + version "2.10.0-6.831-worker-context" + resolved "https://registry.yarnpkg.com/@posthog/node-rdkafka/-/node-rdkafka-2.10.0-6.831-worker-context.tgz#8b361f25f5c7633208331b31bf54583ab4848f61" + integrity sha512-FWzVTFrAeerw9r8QPw+FOjZgyq0GO0Z3z0YzZGHzTWSEZMhpiLRGLiAn73U2OxiKHNFTWqO5Afu6vKoIPwj1kQ== + dependencies: + bindings "^1.3.1" + nan "^2.14.0" + "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" @@ -5134,14 +5142,6 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" -node-rdkafka@^2.9.1: - version "2.9.1" - resolved "https://registry.yarnpkg.com/node-rdkafka/-/node-rdkafka-2.9.1.tgz#ac711bc7845ccb1de87391aec7f27f683d265149" - integrity sha512-C5EVDZlDG+5D8KXiz2zKwEiLWIGW5Z1mkVFRzp13T4mrbXz+ESyjrDSLIj7aoUIi5+T10H9p1wwLZJBh9ivjLg== - dependencies: - bindings "^1.3.1" - nan "^2.14.0" - node-releases@^1.1.65: version "1.1.65" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.65.tgz#52d9579176bd60f23eba05c4438583f341944b81" From 918f9df7ff017a32428a11978035da4d069cc9ce Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 9 Dec 2020 20:51:05 +0100 Subject: [PATCH 054/218] Improve logging --- src/celery/worker.ts | 4 ++-- src/worker/piscina.js | 6 +++--- src/worker/worker.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/celery/worker.ts b/src/celery/worker.ts index 144efa01..49a52e6f 100644 --- a/src/celery/worker.ts +++ b/src/celery/worker.ts @@ -184,11 +184,11 @@ export class Worker extends Base { console.info( `โŒ› ${taskCount} ${taskCount === 1 ? 'task' : 'tasks'} in progress, waiting for ${ taskCount === 1 ? 'it' : 'them' - } to finish before cleaning Celery up...` + } to finish before cleaning up Celery...` ) await this.whenCurrentJobsFinished() } else { - console.info(`๐Ÿ‘ No tasks in progress, cleaning Celery up...`) + console.info(`๐Ÿ‘ No tasks in progress, cleaning up Celery...`) } await this.disconnect() console.info(`๐Ÿ›‘ Celery worker cleaned up!`) diff --git a/src/worker/piscina.js b/src/worker/piscina.js index 3ece3773..b347c63d 100644 --- a/src/worker/piscina.js +++ b/src/worker/piscina.js @@ -1,6 +1,6 @@ const Sentry = require('@sentry/node') -const { isMainThread } = require('worker_threads') +const { isMainThread, threadId } = require('worker_threads') if (isMainThread) { const Piscina = require('piscina') @@ -10,7 +10,7 @@ if (isMainThread) { const piscina = new Piscina(createConfig(serverConfig, __filename)) piscina.on('error', (error) => { Sentry.captureException(error) - console.error('โš ๏ธ Piscina worker error!') + console.error(`โš ๏ธ Piscina worker thread ${threadId} error!`) console.error(error) }) return piscina @@ -23,7 +23,7 @@ if (isMainThread) { const { createWorker } = require('./worker') const { workerData } = require('piscina') - module.exports = createWorker(workerData.serverConfig) + module.exports = createWorker(workerData.serverConfig, threadId) } function areWeTestingWithJest() { diff --git a/src/worker/worker.ts b/src/worker/worker.ts index 1bea8101..832b48de 100644 --- a/src/worker/worker.ts +++ b/src/worker/worker.ts @@ -6,8 +6,8 @@ import { initApp } from '../init' type TaskWorker = ({ task, args }: { task: string; args: any }) => Promise -export async function createWorker(config: PluginsServerConfig): Promise { - console.info('๐Ÿงต Starting Piscina worker threadโ€ฆ') +export async function createWorker(config: PluginsServerConfig, threadId: number): Promise { + console.info(`๐Ÿงต Starting Piscina worker thread ${threadId}โ€ฆ`) initApp(config) From 6662d2e8a1e02a77149d43b450a699d633c14718 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 10 Dec 2020 13:18:35 +0100 Subject: [PATCH 055/218] Unify Redis and Kafka approach to queuing events --- src/celery/worker.ts | 3 +- src/ingestion/process-event.ts | 78 +++++++++++----------------------- src/server.ts | 13 +----- src/types.ts | 4 ++ src/worker/queue.ts | 69 ++++++++++++++++++++++++++++-- 5 files changed, 97 insertions(+), 70 deletions(-) diff --git a/src/celery/worker.ts b/src/celery/worker.ts index 49a52e6f..e7429c40 100644 --- a/src/celery/worker.ts +++ b/src/celery/worker.ts @@ -1,9 +1,10 @@ +import { Queue } from 'types' import Base from './base' import { Message } from './message' type Handler = (...args: any[]) => Promise -export class Worker extends Base { +export class Worker extends Base implements Queue { handlers: Record = {} activeTasks: Set> = new Set() diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 1b21e112..c1a0f321 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,13 +1,22 @@ -import { KafkaConsumer, LibrdKafkaError, Message, Producer, ProducerStream } from '@posthog/node-rdkafka' +import { KafkaConsumer, Producer, ProducerStream } from '@posthog/node-rdkafka' import { DateTime, Duration } from 'luxon' -import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' +import { + PluginsServer, + EventData, + Properties, + Element, + Team, + Person, + PersonDistinctId, + CohortPeople, + Queue, +} from 'types' import { castTimestampOrNow, UUID, UUIDT } from '../utils' -import { KAFKA_EVENTS, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS } from './topics' +import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' -import { runPlugins } from '../plugins' import { Event as EventProto } from '../idl/protos' -export class EventsProcessor { +export class EventsProcessor implements Queue { pluginsServer: PluginsServer kafkaConsumer: KafkaConsumer kafkaProducerStreamEvent: ProducerStream @@ -18,7 +27,16 @@ export class EventsProcessor { throw new Error('You must set KAFKA_HOSTS to process events from Kafka!') } this.pluginsServer = pluginsServer - this.kafkaConsumer = this.buildKafkaConsumer() + this.kafkaConsumer = new KafkaConsumer( + { + 'metadata.broker.list': this.pluginsServer.KAFKA_HOSTS!, + }, + { + 'auto.offset.reset': 'earliest', + } + ).on('disconnected', () => { + console.info(`๐Ÿ›‘ Kafka consumer disconnected!`) + }) this.kafkaProducerStreamEvent = Producer.createWriteStream( { 'metadata.broker.list': pluginsServer.KAFKA_HOSTS, @@ -39,54 +57,6 @@ export class EventsProcessor { ) } - buildKafkaConsumer(): KafkaConsumer { - const kafkaConsumer = new KafkaConsumer( - { - 'metadata.broker.list': this.pluginsServer.KAFKA_HOSTS!, - }, - { - 'auto.offset.reset': 'earliest', - } - ) - - const processBatch = async (error: LibrdKafkaError, messages: Message[]): Promise => { - if (error) { - throw error // TODO: handle errors in a smarter way - } - for (const message of messages) { - const timer = new Date() - let event: EventData | null = JSON.parse(message.value!.toString()) as EventData - event = await runPlugins(this.pluginsServer, event) - if (event) { - await this.process_event_ee( - event.distinct_id, - event.ip, - event.site_url, - event, - event.team_id, - DateTime.fromISO(event.now), - event.sent_at ? DateTime.fromISO(event.sent_at) : null - ) - } - this.pluginsServer.statsd?.timing(`${this.pluginsServer.STATSD_PREFIX}_posthog_cloud`, timer) - } - kafkaConsumer.commit() - } - - kafkaConsumer - .on('ready', () => { - kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) - // consume event messages in batches of 100 - kafkaConsumer.consume(100, processBatch) - console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) - }) - .on('disconnected', () => { - console.info(`๐Ÿ›‘ Kafka consumer disconnected!`) - }) - - return kafkaConsumer - } - /** This must be ran to start consuming events put into Kafka by external web server. */ connectKafkaConsumer(): void { console.info(`โฌ Connecting Kafka consumer...`) diff --git a/src/server.ts b/src/server.ts index 68826fca..65d67cfc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2,17 +2,15 @@ import { Pool } from 'pg' import * as schedule from 'node-schedule' import Redis from 'ioredis' import { FastifyInstance } from 'fastify' -import { PluginsServer, PluginsServerConfig } from './types' +import { PluginsServer, PluginsServerConfig, Queue } from './types' import { startQueue } from './worker/queue' import { startFastifyInstance, stopFastifyInstance } from './web/server' -import { Worker } from 'celery/worker' import { version } from '../package.json' import { PluginEvent } from 'posthog-plugins' import { defaultConfig } from './config' import Piscina from 'piscina' import { StatsD } from 'hot-shots' import * as Sentry from '@sentry/node' -import { EventsProcessor } from './ingestion/process-event' import { delay } from './utils' export async function createServer( @@ -70,11 +68,10 @@ export async function startPluginsServer( let serverConfig: PluginsServerConfig | undefined let pubSub: Redis.Redis | undefined let server: PluginsServer | undefined - let eventsProcessor: EventsProcessor | undefined let fastifyInstance: FastifyInstance | undefined let job: schedule.Job | undefined let piscina: Piscina | undefined - let queue: Worker | undefined + let queue: Queue | undefined let closeServer: () => Promise | undefined let shuttingDown = false @@ -84,7 +81,6 @@ export async function startPluginsServer( return } shuttingDown = true - eventsProcessor?.stop() if (fastifyInstance && !serverConfig?.DISABLE_WEB) { await stopFastifyInstance(fastifyInstance!) } @@ -114,11 +110,6 @@ export async function startPluginsServer( piscina = makePiscina(serverConfig) const processEvent = (event: PluginEvent) => piscina!.runTask({ task: 'processEvent', args: { event } }) - if (serverConfig.EE_ENABLED) { - eventsProcessor = new EventsProcessor(server) - eventsProcessor.connectKafkaConsumer() - } - if (!serverConfig.DISABLE_WEB) { fastifyInstance = await startFastifyInstance(server) } diff --git a/src/types.ts b/src/types.ts index 71021ec2..5326cea7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -53,6 +53,10 @@ export interface PluginsServer extends PluginsServerConfig { defaultConfigs: PluginConfig[] } +export interface Queue { + stop: () => void +} + export type PluginId = number export type PluginConfigId = number export type TeamId = number diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 00ff3027..972d3eb9 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -1,14 +1,17 @@ -import { PluginsServer } from '../types' import { PluginEvent } from 'posthog-plugins' import * as Sentry from '@sentry/node' - +import { LibrdKafkaError, Message } from '@posthog/node-rdkafka' +import { DateTime } from 'luxon' import Worker from '../celery/worker' import Client from '../celery/client' +import { EventData, PluginsServer, Queue } from '../types' +import { EventsProcessor } from '../ingestion/process-event' +import { KAFKA_EVENTS_WAL } from '../ingestion/topics' -export function startQueue( +function startQueueRedis( server: PluginsServer, processEvent: (event: PluginEvent) => Promise -): Worker { +): Queue { const worker = new Worker(server.redis, server.PLUGINS_CELERY_QUEUE) const client = new Client(server.redis, server.CELERY_DEFAULT_QUEUE) @@ -48,3 +51,61 @@ export function startQueue( return worker } + +function startQueueKafka( + server: PluginsServer, + processEvent: (event: PluginEvent) => Promise +): Queue { + const eventsProcessor = new EventsProcessor(server) + eventsProcessor.connectKafkaConsumer() + + eventsProcessor.kafkaConsumer.on('ready', () => { + console.info(`โœ… Kafka consumer ready!`) + eventsProcessor.kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) + // consume event messages in batches of 100 + eventsProcessor.kafkaConsumer.consume( + 100, + async (error: LibrdKafkaError, messages: Message[]): Promise => { + if (error) { + Sentry.captureException(error) + } + try { + for (const message of messages) { + const timer = new Date() + const event = JSON.parse(message.value!.toString()) as EventData + const processedEvent = await processEvent(event) + if (processedEvent) { + const { distinct_id, ip, site_url, team_id, now, sent_at } = processedEvent + await eventsProcessor.process_event_ee( + distinct_id, + ip, + site_url, + processedEvent, + team_id, + DateTime.fromISO(now), + sent_at ? DateTime.fromISO(sent_at) : null + ) + } + server.statsd?.timing(`${server.STATSD_PREFIX}_posthog_cloud`, timer) + } + eventsProcessor.kafkaConsumer.commit() + } catch (error) { + Sentry.captureException(error) + } + } + ) + console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) + }) + + eventsProcessor.connectKafkaConsumer() + + return eventsProcessor +} + +export function startQueue( + server: PluginsServer, + processEvent: (event: PluginEvent) => Promise +): Queue { + const relevantStartQueue = server.EE_ENABLED ? startQueueKafka : startQueueRedis + return relevantStartQueue(server, processEvent) +} From 427b89fa8fe68065b31b945fd28b428423737f5e Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 10 Dec 2020 16:23:29 +0100 Subject: [PATCH 056/218] Fix consuming from Kafka --- src/ingestion/process-event.ts | 3 +- src/ingestion/topics.ts | 1 + src/worker/queue.ts | 73 ++++++++++++++++++---------------- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index c1a0f321..7592127e 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -29,6 +29,7 @@ export class EventsProcessor implements Queue { this.pluginsServer = pluginsServer this.kafkaConsumer = new KafkaConsumer( { + 'group.id': 'whatever', // TODO: make this something meaningful other than "whatever"? I don't know 'metadata.broker.list': this.pluginsServer.KAFKA_HOSTS!, }, { @@ -59,7 +60,7 @@ export class EventsProcessor implements Queue { /** This must be ran to start consuming events put into Kafka by external web server. */ connectKafkaConsumer(): void { - console.info(`โฌ Connecting Kafka consumer...`) + console.info(`โฌ Connecting Kafka consumer to ${this.pluginsServer.KAFKA_HOSTS}...`) this.kafkaConsumer.connect() } diff --git a/src/ingestion/topics.ts b/src/ingestion/topics.ts index c90ebf55..206d95e7 100644 --- a/src/ingestion/topics.ts +++ b/src/ingestion/topics.ts @@ -3,3 +3,4 @@ export const KAFKA_PERSON = 'clickhouse_person' export const KAFKA_PERSON_UNIQUE_ID = 'clickhouse_person_unique_id' export const KAFKA_SESSION_RECORDING_EVENTS = 'clickhouse_session_recording_events' export const KAFKA_EVENTS_WAL = 'events_write_ahead_log' +export const KAFKA_EVENTS_HANDOFF = 'events_handoff' diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 972d3eb9..f66ae521 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -6,7 +6,7 @@ import Worker from '../celery/worker' import Client from '../celery/client' import { EventData, PluginsServer, Queue } from '../types' import { EventsProcessor } from '../ingestion/process-event' -import { KAFKA_EVENTS_WAL } from '../ingestion/topics' +import { KAFKA_EVENTS_HANDOFF } from '../ingestion/topics' function startQueueRedis( server: PluginsServer, @@ -57,44 +57,49 @@ function startQueueKafka( processEvent: (event: PluginEvent) => Promise ): Queue { const eventsProcessor = new EventsProcessor(server) - eventsProcessor.connectKafkaConsumer() eventsProcessor.kafkaConsumer.on('ready', () => { - console.info(`โœ… Kafka consumer ready!`) - eventsProcessor.kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) - // consume event messages in batches of 100 - eventsProcessor.kafkaConsumer.consume( - 100, - async (error: LibrdKafkaError, messages: Message[]): Promise => { - if (error) { - Sentry.captureException(error) - } - try { - for (const message of messages) { - const timer = new Date() - const event = JSON.parse(message.value!.toString()) as EventData - const processedEvent = await processEvent(event) - if (processedEvent) { - const { distinct_id, ip, site_url, team_id, now, sent_at } = processedEvent - await eventsProcessor.process_event_ee( - distinct_id, - ip, - site_url, - processedEvent, - team_id, - DateTime.fromISO(now), - sent_at ? DateTime.fromISO(sent_at) : null - ) + eventsProcessor.kafkaConsumer.subscribe([KAFKA_EVENTS_HANDOFF]) + // consume event messages in batches of 100 every 100 ms + setInterval(() => { + eventsProcessor.kafkaConsumer.consume( + 100, + async (error: LibrdKafkaError, messages: Message[]): Promise => { + if (messages.length) { + console.info( + `${messages.length} ${messages.length === 1 ? 'message' : 'messages'} consumed from Kafka` + ) + } + if (error) { + Sentry.captureException(error) + } + try { + for (const message of messages) { + const timer = new Date() + const event = JSON.parse(message.value!.toString()) as EventData + const processedEvent = await processEvent(event) + if (processedEvent) { + const { distinct_id, ip, site_url, team_id, now, sent_at } = processedEvent + await eventsProcessor.process_event_ee( + distinct_id, + ip, + site_url, + processedEvent, + team_id, + DateTime.fromISO(now), + sent_at ? DateTime.fromISO(sent_at) : null + ) + } + server.statsd?.timing(`${server.STATSD_PREFIX}_posthog_cloud`, timer) } - server.statsd?.timing(`${server.STATSD_PREFIX}_posthog_cloud`, timer) + eventsProcessor.kafkaConsumer.commit() + } catch (error) { + Sentry.captureException(error) } - eventsProcessor.kafkaConsumer.commit() - } catch (error) { - Sentry.captureException(error) } - } - ) - console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) + ) + }, 100) + console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_HANDOFF}!`) }) eventsProcessor.connectKafkaConsumer() From 03b9eefeec021afa73501638c5a9acad035abc2e Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 10 Dec 2020 16:59:59 +0100 Subject: [PATCH 057/218] Make some optimizations --- src/ingestion/process-event.ts | 51 +++++++++++++++++----------------- src/server.ts | 6 ++-- src/worker/queue.ts | 2 +- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 7592127e..c0fc96da 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -15,9 +15,11 @@ import { castTimestampOrNow, UUID, UUIDT } from '../utils' import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elements_to_string } from './element' import { Event as EventProto } from '../idl/protos' +import { Pool } from 'pg' export class EventsProcessor implements Queue { pluginsServer: PluginsServer + db: Pool kafkaConsumer: KafkaConsumer kafkaProducerStreamEvent: ProducerStream kafkaProducerStreamSessionRecording: ProducerStream @@ -27,6 +29,7 @@ export class EventsProcessor implements Queue { throw new Error('You must set KAFKA_HOSTS to process events from Kafka!') } this.pluginsServer = pluginsServer + this.db = pluginsServer.db this.kafkaConsumer = new KafkaConsumer( { 'group.id': 'whatever', // TODO: make this something meaningful other than "whatever"? I don't know @@ -157,7 +160,7 @@ export class EventsProcessor implements Queue { async _set_is_identified(team_id: number, distinct_id: string, is_identified = true): Promise { let personFound: Person | undefined personFound = ( - await this.pluginsServer.db.query( + await this.db.query( 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', [team_id, distinct_id] ) @@ -169,7 +172,7 @@ export class EventsProcessor implements Queue { // Catch race condition where in between getting and creating, another request already created this person } catch { personFound = ( - await this.pluginsServer.db.query( + await this.db.query( 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', [team_id, distinct_id] ) @@ -177,16 +180,14 @@ export class EventsProcessor implements Queue { } } if (personFound && !personFound.is_identified) { - await this.pluginsServer.db.query('UPDATE posthog_person SET is_identified = 1 WHERE id = $1', [ - personFound.id, - ]) + await this.db.query('UPDATE posthog_person SET is_identified = 1 WHERE id = $1', [personFound.id]) } } async _update_person_properties(team_id: number, distinct_id: string, properties: Properties): Promise { let personFound: Person | undefined personFound = ( - await this.pluginsServer.db.query( + await this.db.query( 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', [team_id, distinct_id] ) @@ -205,7 +206,7 @@ export class EventsProcessor implements Queue { // Catch race condition where in between getting and creating, another request already created this person } catch { personFound = ( - await this.pluginsServer.db.query( + await this.db.query( 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', [team_id, distinct_id] ) @@ -213,7 +214,7 @@ export class EventsProcessor implements Queue { } } if (personFound) { - this.pluginsServer.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ + this.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ { ...personFound.properties, ...properties }, personFound.id, ]) @@ -227,14 +228,14 @@ export class EventsProcessor implements Queue { retry_if_failed = true ): Promise { const old_person: Person | undefined = ( - await this.pluginsServer.db.query( + await this.db.query( 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', [team_id, previous_distinct_id] ) ).rows[0] const new_person: Person | undefined = ( - await this.pluginsServer.db.query( + await this.db.query( 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', [team_id, distinct_id] ) @@ -301,7 +302,7 @@ export class EventsProcessor implements Queue { } } - await this.pluginsServer.db.query('UPDATE posthog_person SET created_at = $1 WHERE id = $2', [ + await this.db.query('UPDATE posthog_person SET created_at = $1 WHERE id = $2', [ first_seen.toISO(), merge_into.id, ]) @@ -309,31 +310,29 @@ export class EventsProcessor implements Queue { // merge the distinct_ids for (const other_person of people_to_merge) { const other_person_distinct_ids: PersonDistinctId[] = ( - await this.pluginsServer.db.query( - 'SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', - [other_person, merge_into.team_id] - ) + await this.db.query('SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', [ + other_person, + merge_into.team_id, + ]) ).rows for (const person_distinct_id of other_person_distinct_ids) { - await this.pluginsServer.db.query('UPDATE posthog_persondistinctid SET person_id = $1 WHERE id = $2', [ + await this.db.query('UPDATE posthog_persondistinctid SET person_id = $1 WHERE id = $2', [ merge_into.id, person_distinct_id.id, ]) } const other_person_cohort_ids: CohortPeople[] = ( - await this.pluginsServer.db.query('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [ - other_person.id, - ]) + await this.db.query('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [other_person.id]) ).rows for (const person_cohort_id of other_person_cohort_ids) { - await this.pluginsServer.db.query('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ + await this.db.query('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ merge_into.id, person_cohort_id.id, ]) } - await this.pluginsServer.db.query('DELETE FROM posthog_person WHERE id = $1', [other_person.id]) + await this.db.query('DELETE FROM posthog_person WHERE id = $1', [other_person.id]) } } @@ -368,7 +367,7 @@ export class EventsProcessor implements Queue { rows: [team], }: { rows: Team[] - } = await this.pluginsServer.db.query( + } = await this.db.query( 'SELECT slack_incoming_webhook, event_names, event_properties, event_names_with_usage, event_properties_with_usage, anonymize_ips FROM posthog_team WHERE id = $1', [team_id] ) @@ -383,7 +382,7 @@ export class EventsProcessor implements Queue { rows: [{ pdiCount }], }: { rows: { pdiCount: number }[] - } = await this.pluginsServer.db.query( + } = await this.db.query( 'SELECT COUNT(*) AS pdiCount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', [team_id, distinct_id] ) @@ -432,7 +431,7 @@ export class EventsProcessor implements Queue { } } if (save) { - await this.pluginsServer.db.query( + await this.db.query( 'UPDATE posthog_team SET ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, event_properties_with_usage = $5, event_properties_numerical = $6 WHERE id = $7', [ team.ingested_event, @@ -456,7 +455,7 @@ export class EventsProcessor implements Queue { uuid: UUID | string ): Promise { return ( - await this.pluginsServer.db.query( + await this.db.query( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6)', [created_at.toISO(), properties, team_id, is_user_id, is_identified, uuid.toString()] ) @@ -464,7 +463,7 @@ export class EventsProcessor implements Queue { } async add_distinct_id(person: Person, distinct_id: string): Promise { - await this.pluginsServer.db.query( + await this.db.query( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3)', [distinct_id, person.id, person.team_id] ) diff --git a/src/server.ts b/src/server.ts index 2ba875c9..a4851488 100644 --- a/src/server.ts +++ b/src/server.ts @@ -24,7 +24,7 @@ export async function createServer( const redis = new Redis(serverConfig.REDIS_URL, { maxRetriesPerRequest: -1 }) redis.on('error', (error) => { Sentry.captureException(error) - console.error('๐Ÿ”ด Redis error! Trying to reconnect.') + console.error('๐Ÿ”ด Redis error! Trying to reconnect...') console.error(error) }) await redis.info() @@ -130,8 +130,8 @@ export async function startPluginsServer( // every 5 sec set a @posthog-plugin-server/ping redis key job = schedule.scheduleJob('*/5 * * * * *', () => { - server!.redis!.set('@posthog-plugin-server/ping', new Date().toISOString()) - server!.redis!.expire('@posthog-plugin-server/ping', 60) + server!.redis.set('@posthog-plugin-server/ping', new Date().toISOString()) + server!.redis.expire('@posthog-plugin-server/ping', 60) }) console.info(`๐Ÿš€ All systems go.`) } catch (error) { diff --git a/src/worker/queue.ts b/src/worker/queue.ts index f66ae521..61e4613b 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -65,7 +65,7 @@ function startQueueKafka( eventsProcessor.kafkaConsumer.consume( 100, async (error: LibrdKafkaError, messages: Message[]): Promise => { - if (messages.length) { + if (messages?.length) { console.info( `${messages.length} ${messages.length === 1 ? 'message' : 'messages'} consumed from Kafka` ) From f155291c8195ebb94a060a2250cfa7b4e887b4cc Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 10 Dec 2020 23:31:43 +0100 Subject: [PATCH 058/218] Don't introduce KAFKA_EVENTS_HANDOFF --- src/ingestion/process-event.ts | 2 +- src/ingestion/topics.ts | 1 - src/worker/queue.ts | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index c0fc96da..e5925c1d 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -32,7 +32,7 @@ export class EventsProcessor implements Queue { this.db = pluginsServer.db this.kafkaConsumer = new KafkaConsumer( { - 'group.id': 'whatever', // TODO: make this something meaningful other than "whatever"? I don't know + 'group.id': 'plugin-ingestion', 'metadata.broker.list': this.pluginsServer.KAFKA_HOSTS!, }, { diff --git a/src/ingestion/topics.ts b/src/ingestion/topics.ts index 206d95e7..c90ebf55 100644 --- a/src/ingestion/topics.ts +++ b/src/ingestion/topics.ts @@ -3,4 +3,3 @@ export const KAFKA_PERSON = 'clickhouse_person' export const KAFKA_PERSON_UNIQUE_ID = 'clickhouse_person_unique_id' export const KAFKA_SESSION_RECORDING_EVENTS = 'clickhouse_session_recording_events' export const KAFKA_EVENTS_WAL = 'events_write_ahead_log' -export const KAFKA_EVENTS_HANDOFF = 'events_handoff' diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 61e4613b..dfb1ff93 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -6,7 +6,7 @@ import Worker from '../celery/worker' import Client from '../celery/client' import { EventData, PluginsServer, Queue } from '../types' import { EventsProcessor } from '../ingestion/process-event' -import { KAFKA_EVENTS_HANDOFF } from '../ingestion/topics' +import { KAFKA_EVENTS_WAL } from '../ingestion/topics' function startQueueRedis( server: PluginsServer, @@ -59,11 +59,11 @@ function startQueueKafka( const eventsProcessor = new EventsProcessor(server) eventsProcessor.kafkaConsumer.on('ready', () => { - eventsProcessor.kafkaConsumer.subscribe([KAFKA_EVENTS_HANDOFF]) + eventsProcessor.kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) // consume event messages in batches of 100 every 100 ms setInterval(() => { eventsProcessor.kafkaConsumer.consume( - 100, + 10000, async (error: LibrdKafkaError, messages: Message[]): Promise => { if (messages?.length) { console.info( @@ -99,7 +99,7 @@ function startQueueKafka( } ) }, 100) - console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_HANDOFF}!`) + console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) }) eventsProcessor.connectKafkaConsumer() From 5dc876050cc21b788895892bb459da47e9c50169 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 11 Dec 2020 10:31:30 +0100 Subject: [PATCH 059/218] Consume 1 kiloevent / 50 ms = 20 kiloevents / s --- src/worker/queue.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index dfb1ff93..f853dfc3 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -60,10 +60,10 @@ function startQueueKafka( eventsProcessor.kafkaConsumer.on('ready', () => { eventsProcessor.kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) - // consume event messages in batches of 100 every 100 ms + // consume event messages in batches of 1000 every 50 ms setInterval(() => { eventsProcessor.kafkaConsumer.consume( - 10000, + 1000, async (error: LibrdKafkaError, messages: Message[]): Promise => { if (messages?.length) { console.info( @@ -98,7 +98,7 @@ function startQueueKafka( } } ) - }, 100) + }, 50) console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) }) From 35ecf02d9ac1116d8d71747507c6ee75f039950a Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 11 Dec 2020 11:20:21 +0100 Subject: [PATCH 060/218] Don't start web server by default We don't need it at all yet. --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index de260891..49f3467a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,7 +13,7 @@ export function getDefaultConfig(): PluginsServerConfig { REDIS_URL: 'redis://localhost/', BASE_DIR: '.', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'reload-plugins', - DISABLE_WEB: false, + DISABLE_WEB: true, WEB_PORT: 3008, WEB_HOSTNAME: '0.0.0.0', WORKER_CONCURRENCY: 0, // use all cores From f851efa4c85b5abd82fc2862c93b1a6d6bf743e4 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 11 Dec 2020 11:22:05 +0100 Subject: [PATCH 061/218] Improve Redis logging --- src/server.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/server.ts b/src/server.ts index a42a76bc..797c8acd 100644 --- a/src/server.ts +++ b/src/server.ts @@ -22,11 +22,15 @@ export async function createServer( } const redis = new Redis(serverConfig.REDIS_URL, { maxRetriesPerRequest: -1 }) - redis.on('error', (error) => { - Sentry.captureException(error) - console.error('๐Ÿ”ด Redis error! Trying to reconnect...') - console.error(error) - }) + redis + .on('error', (error) => { + Sentry.captureException(error) + console.error('๐Ÿ”ด Redis error encountered! Trying to reconnect...') + console.error(error) + }) + .on('ready', () => { + console.info(`โœ… Connected to Redis at ${serverConfig.REDIS_URL}!`) + }) await redis.info() const db = new Pool({ From b4907ca66526409ba38fe8e0e852012c62a77ed9 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 11 Dec 2020 11:22:29 +0100 Subject: [PATCH 062/218] Fix connecting to dockerized Redis No idea why this fixes the issue, but it does. --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 49f3467a..72cd2623 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,7 +10,7 @@ export function getDefaultConfig(): PluginsServerConfig { KAFKA_HOSTS: null, EE_ENABLED: false, PLUGINS_CELERY_QUEUE: 'posthog-plugins', - REDIS_URL: 'redis://localhost/', + REDIS_URL: 'redis://127.0.0.1', BASE_DIR: '.', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'reload-plugins', DISABLE_WEB: true, From 15215ef33c9cb36cae724d2b8f2ab0770e8b0026 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 11 Dec 2020 12:21:18 +0100 Subject: [PATCH 063/218] Update yarn.lock --- yarn.lock | 902 +++++++++++++++++++++++------------------------------- 1 file changed, 387 insertions(+), 515 deletions(-) diff --git a/yarn.lock b/yarn.lock index cb99c2d8..ed27cf0a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8,9 +8,9 @@ integrity sha512-H71nDOOL8Y7kWRLqf6Sums+01Q5msqBW2KhDUTemh1tvY04eSkSXrK0uj/4mmY0Xr16/3zyZmsrxN7CKuRbNRg== "@babel/cli@^7.0.0": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.12.1.tgz#e08a0b1cb6fcd4b9eb6a606ba5602c5c0fe24a0c" - integrity sha512-eRJREyrfAJ2r42Iaxe8h3v6yyj1wu9OyosaUHW6UImjGf9ahGL9nsFNh7OCopvtcPL8WnEo7tp78wrZaZ6vG9g== + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.12.10.tgz#67a1015b1cd505bde1696196febf910c4c339a48" + integrity sha512-+y4ZnePpvWs1fc/LhZRTHkTesbXkyBYuOB+5CyodZqrEuETXi3zOVfpAQIdgC3lXbHLTDG9dQosxR9BhvLKDLQ== dependencies: commander "^4.0.1" convert-source-map "^1.1.0" @@ -21,7 +21,7 @@ slash "^2.0.0" source-map "^0.5.0" optionalDependencies: - "@nicolo-ribaudo/chokidar-2" "^2.1.8" + "@nicolo-ribaudo/chokidar-2" "2.1.8-no-fsevents" chokidar "^3.4.0" "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4": @@ -31,70 +31,47 @@ dependencies: "@babel/highlight" "^7.10.4" -"@babel/compat-data@^7.12.1", "@babel/compat-data@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.12.5.tgz#f56db0c4bb1bbbf221b4e81345aab4141e7cb0e9" - integrity sha512-DTsS7cxrsH3by8nqQSpFSyjSfSYl57D6Cf4q8dW3LK83tBKBDCkfcay1nYkXq1nIHXnpX8WMMb/O25HOy3h1zg== - -"@babel/core@^7.0.0": - version "7.12.3" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.3.tgz#1b436884e1e3bff6fb1328dc02b208759de92ad8" - integrity sha512-0qXcZYKZp3/6N2jKYVxZv0aNCsxTSVCiK72DTiTYZAu7sjg73W0/aynWjMbiGd87EQL4WyA8reiJVh92AVla9g== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.1" - "@babel/helper-module-transforms" "^7.12.1" - "@babel/helpers" "^7.12.1" - "@babel/parser" "^7.12.3" - "@babel/template" "^7.10.4" - "@babel/traverse" "^7.12.1" - "@babel/types" "^7.12.1" - convert-source-map "^1.7.0" - debug "^4.1.0" - gensync "^1.0.0-beta.1" - json5 "^2.1.2" - lodash "^4.17.19" - resolve "^1.3.2" - semver "^5.4.1" - source-map "^0.5.0" +"@babel/compat-data@^7.12.5", "@babel/compat-data@^7.12.7": + version "7.12.7" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.12.7.tgz#9329b4782a7d6bbd7eef57e11addf91ee3ef1e41" + integrity sha512-YaxPMGs/XIWtYqrdEOZOCPsVWfEoriXopnsz3/i7apYPXQ3698UFhS6dVT1KN5qOsWmVgw/FOrmQgpRaZayGsw== -"@babel/core@^7.1.0", "@babel/core@^7.7.5": - version "7.12.9" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.9.tgz#fd450c4ec10cdbb980e2928b7aa7a28484593fc8" - integrity sha512-gTXYh3M5wb7FRXQy+FErKFAv90BnlOuNn1QkCK2lREoPAjrQCO49+HVSrFoe5uakFAF5eenS75KbO2vQiLrTMQ== +"@babel/core@^7.0.0", "@babel/core@^7.1.0", "@babel/core@^7.7.5": + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.10.tgz#b79a2e1b9f70ed3d84bbfb6d8c4ef825f606bccd" + integrity sha512-eTAlQKq65zHfkHZV0sIVODCPGVgoo1HdBlbSLi9CqOzuZanMv2ihzY+4paiKr1mH+XmYESMAmJ/dpZ68eN6d8w== dependencies: "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.5" + "@babel/generator" "^7.12.10" "@babel/helper-module-transforms" "^7.12.1" "@babel/helpers" "^7.12.5" - "@babel/parser" "^7.12.7" + "@babel/parser" "^7.12.10" "@babel/template" "^7.12.7" - "@babel/traverse" "^7.12.9" - "@babel/types" "^7.12.7" + "@babel/traverse" "^7.12.10" + "@babel/types" "^7.12.10" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.1" json5 "^2.1.2" lodash "^4.17.19" - resolve "^1.3.2" semver "^5.4.1" source-map "^0.5.0" -"@babel/generator@^7.12.1", "@babel/generator@^7.12.5": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.5.tgz#a2c50de5c8b6d708ab95be5e6053936c1884a4de" - integrity sha512-m16TQQJ8hPt7E+OS/XVQg/7U184MLXtvuGbCdA7na61vha+ImkyyNM/9DDA0unYCVZn3ZOhng+qz48/KBOT96A== +"@babel/generator@^7.12.10": + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.10.tgz#2b188fc329fb8e4f762181703beffc0fe6df3460" + integrity sha512-6mCdfhWgmqLdtTkhXjnIz0LcdVCd26wS2JXRtj2XY0u5klDsXBREA/pG5NVOuVnF2LUrBGNFtQkIqqTbblg0ww== dependencies: - "@babel/types" "^7.12.5" + "@babel/types" "^7.12.10" jsesc "^2.5.1" source-map "^0.5.0" "@babel/helper-annotate-as-pure@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.10.4.tgz#5bf0d495a3f757ac3bda48b5bf3b3ba309c72ba3" - integrity sha512-XQlqKQP4vXFB7BN8fEEerrmYvHp3fK/rBkRFz9jaJbzK0B1DSfej9Kc7ZzE8Z/OnId1jpJdNAZ3BFQjWG68rcA== + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.10.tgz#54ab9b000e60a93644ce17b3f37d313aaf1d115d" + integrity sha512-XplmVbC1n+KY6jL8/fgLVXXUauDIB+lD5+GsQEh6F6GBF1dq1qy4DP4yXWzDKcoqXB3X58t61e85Fitoww4JVQ== dependencies: - "@babel/types" "^7.10.4" + "@babel/types" "^7.12.10" "@babel/helper-builder-binary-assignment-operator-visitor@^7.10.4": version "7.10.4" @@ -104,7 +81,7 @@ "@babel/helper-explode-assignable-expression" "^7.10.4" "@babel/types" "^7.10.4" -"@babel/helper-compilation-targets@^7.12.1": +"@babel/helper-compilation-targets@^7.12.5": version "7.12.5" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.12.5.tgz#cb470c76198db6a24e9dbc8987275631e5d29831" integrity sha512-+qH6NrscMolUlzOYngSBMIOQpKUGPPsc61Bu5W10mg84LxZ7cmvnBHzARKbDoFxVvqqAbj6Tg6N7bSrWSPXMyw== @@ -126,12 +103,11 @@ "@babel/helper-split-export-declaration" "^7.10.4" "@babel/helper-create-regexp-features-plugin@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.1.tgz#18b1302d4677f9dc4740fe8c9ed96680e29d37e8" - integrity sha512-rsZ4LGvFTZnzdNZR5HZdmJVuXK8834R5QkF3WvcnBhrlVtF0HSIUC6zbreL9MgjTywhKokn8RIYRiq99+DLAxA== + version "7.12.7" + resolved "https://registry.yarnpkg.com/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.7.tgz#2084172e95443fa0a09214ba1bb328f9aea1278f" + integrity sha512-idnutvQPdpbduutvi3JVfEgcVIHooQnhvhx0Nk9isOINOIGYkZea1Pk2JlJRiUnMefrlvr0vkByATBY/mB4vjQ== dependencies: "@babel/helper-annotate-as-pure" "^7.10.4" - "@babel/helper-regex" "^7.10.4" regexpu-core "^4.7.1" "@babel/helper-define-map@^7.10.4": @@ -160,11 +136,11 @@ "@babel/types" "^7.10.4" "@babel/helper-get-function-arity@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz#98c1cbea0e2332f33f9a4661b8ce1505b2c19ba2" - integrity sha512-EkN3YDB+SRDgiIUnNgcmiD361ti+AVbL3f3Henf6dqqUyr5dMsorno0lJWJuLhDhkI5sYEpgj6y9kB8AOU1I2A== + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.10.tgz#b158817a3165b5faa2047825dfa61970ddcc16cf" + integrity sha512-mm0n5BPjR06wh9mPQaDdXWDoll/j5UpCAPl1x8fS71GHm7HA6Ua2V4ylG1Ju8lvcTOietbPNNPaSilKj+pj+Ag== dependencies: - "@babel/types" "^7.10.4" + "@babel/types" "^7.12.10" "@babel/helper-hoist-variables@^7.10.4": version "7.10.4" @@ -174,13 +150,13 @@ "@babel/types" "^7.10.4" "@babel/helper-member-expression-to-functions@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.1.tgz#fba0f2fcff3fba00e6ecb664bb5e6e26e2d6165c" - integrity sha512-k0CIe3tXUKTRSoEx1LQEPFU9vRQfqHtl+kf8eNnDqb4AUJEy5pz6aIiog+YWtVm2jpggjS1laH68bPsR+KWWPQ== + version "7.12.7" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.7.tgz#aa77bd0396ec8114e5e30787efa78599d874a855" + integrity sha512-DCsuPyeWxeHgh1Dus7APn7iza42i/qXqiFPWyBDdOFtvS581JQePsc1F/nD+fHrcswhLlRc2UpYS1NwERxZhHw== dependencies: - "@babel/types" "^7.12.1" + "@babel/types" "^7.12.7" -"@babel/helper-module-imports@^7.12.1": +"@babel/helper-module-imports@^7.12.1", "@babel/helper-module-imports@^7.12.5": version "7.12.5" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz#1bfc0229f794988f76ed0a4d4e90860850b54dfb" integrity sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA== @@ -203,24 +179,17 @@ lodash "^4.17.19" "@babel/helper-optimise-call-expression@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.4.tgz#50dc96413d594f995a77905905b05893cd779673" - integrity sha512-n3UGKY4VXwXThEiKrgRAoVPBMqeoPgHVqiHZOanAJCG9nQUL2pLRQirUzl0ioKclHGpGqRgIOkgcIJaIWLpygg== + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.10.tgz#94ca4e306ee11a7dd6e9f42823e2ac6b49881e2d" + integrity sha512-4tpbU0SrSTjjt65UMWSrUOPZTsgvPgGG4S8QSTNHacKzpS51IVWGDj0yCwyeZND/i+LSN2g/O63jEXEWm49sYQ== dependencies: - "@babel/types" "^7.10.4" + "@babel/types" "^7.12.10" "@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.8.0", "@babel/helper-plugin-utils@^7.8.3": version "7.10.4" resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz#2f75a831269d4f677de49986dff59927533cf375" integrity sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg== -"@babel/helper-regex@^7.10.4": - version "7.10.5" - resolved "https://registry.yarnpkg.com/@babel/helper-regex/-/helper-regex-7.10.5.tgz#32dfbb79899073c415557053a19bd055aae50ae0" - integrity sha512-68kdUAzDrljqBrio7DYAEgCoJHxppJOERHOgOrDN7WjOzP0ZQ1LsSDRXcemzVZaLvjaJsJEESb6qt+znNuENDg== - dependencies: - lodash "^4.17.19" - "@babel/helper-remap-async-to-generator@^7.12.1": version "7.12.1" resolved "https://registry.yarnpkg.com/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.12.1.tgz#8c4dbbf916314f6047dc05e6a2217074238347fd" @@ -281,7 +250,7 @@ "@babel/traverse" "^7.10.4" "@babel/types" "^7.10.4" -"@babel/helpers@^7.12.1", "@babel/helpers@^7.12.5": +"@babel/helpers@^7.12.5": version "7.12.5" resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.12.5.tgz#1a1ba4a768d9b58310eda516c449913fe647116e" integrity sha512-lgKGMQlKqA8meJqKsW6rUnc4MdUk35Ln0ATDqdM1a/UpARODdI4j5Y5lVfUScnSNkJcdCRAaWkspykNoFg9sJA== @@ -299,15 +268,10 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.12.7": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.7.tgz#fee7b39fe809d0e73e5b25eecaf5780ef3d73056" - integrity sha512-oWR02Ubp4xTLCAqPRiNIuMVgNO5Aif/xpXtabhzW2HWUD47XJsAB4Zd/Rg30+XeQA3juXigV7hlquOTmwqLiwg== - -"@babel/parser@^7.10.4", "@babel/parser@^7.12.3", "@babel/parser@^7.12.5", "@babel/parser@^7.7.0": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.5.tgz#b4af32ddd473c0bfa643bd7ff0728b8e71b81ea0" - integrity sha512-FVM6RZQ0mn2KCf1VUED7KepYeUWoVShczewOCfm3nzoBybaih51h+sYVVGthW9M6lPByEPTQf+xm27PBdlpwmQ== +"@babel/parser@^7.1.0", "@babel/parser@^7.12.10", "@babel/parser@^7.12.7", "@babel/parser@^7.7.0": + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.10.tgz#824600d59e96aea26a5a2af5a9d812af05c3ae81" + integrity sha512-PJdRPwyoOqFAWfLytxrWwGrAxghCgh/yTNCYciOz8QgjflA7aZhECPZAa2VUedKg2+QMWkI0L9lynh2SNmNEgA== "@babel/plugin-proposal-async-generator-functions@^7.12.1": version "7.12.1" @@ -366,10 +330,10 @@ "@babel/helper-plugin-utils" "^7.10.4" "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.0" -"@babel/plugin-proposal-numeric-separator@^7.12.1": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.12.5.tgz#b1ce757156d40ed79d59d467cb2b154a5c4149ba" - integrity sha512-UiAnkKuOrCyjZ3sYNHlRlfuZJbBHknMQ9VMwVeX97Ofwx7RpD6gS2HfqTCh8KNUQgcOm8IKt103oR4KIjh7Q8g== +"@babel/plugin-proposal-numeric-separator@^7.12.7": + version "7.12.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.12.7.tgz#8bf253de8139099fea193b297d23a9d406ef056b" + integrity sha512-8c+uy0qmnRTeukiGsjLGy6uVs/TFjJchGXUeBqlG4VWYOdJWkhhVPdQ3uHwbmalfJwv2JsV0qffXP4asRfL2SQ== dependencies: "@babel/helper-plugin-utils" "^7.10.4" "@babel/plugin-syntax-numeric-separator" "^7.10.4" @@ -391,10 +355,10 @@ "@babel/helper-plugin-utils" "^7.10.4" "@babel/plugin-syntax-optional-catch-binding" "^7.8.0" -"@babel/plugin-proposal-optional-chaining@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.12.1.tgz#cce122203fc8a32794296fc377c6dedaf4363797" - integrity sha512-c2uRpY6WzaVDzynVY9liyykS+kVU+WRZPMPYpkelXH8KBt1oXoI89kPbZKKG/jDT5UK92FTW2fZkZaJhdiBabw== +"@babel/plugin-proposal-optional-chaining@^7.12.7": + version "7.12.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.12.7.tgz#e02f0ea1b5dc59d401ec16fb824679f683d3303c" + integrity sha512-4ovylXZ0PWmwoOvhU2vhnzVNnm88/Sm9nx7V8BPgMvAzn5zDou3/Awy0EjglyubVHasJj+XCEkr/r1X3P5elCA== dependencies: "@babel/helper-plugin-utils" "^7.10.4" "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1" @@ -734,13 +698,12 @@ "@babel/helper-plugin-utils" "^7.10.4" "@babel/helper-skip-transparent-expression-wrappers" "^7.12.1" -"@babel/plugin-transform-sticky-regex@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.12.1.tgz#5c24cf50de396d30e99afc8d1c700e8bce0f5caf" - integrity sha512-CiUgKQ3AGVk7kveIaPEET1jNDhZZEl1RPMWdTBE1799bdz++SwqDHStmxfCtDfBhQgCl38YRiSnrMuUMZIWSUQ== +"@babel/plugin-transform-sticky-regex@^7.12.7": + version "7.12.7" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.12.7.tgz#560224613ab23987453948ed21d0b0b193fa7fad" + integrity sha512-VEiqZL5N/QvDbdjfYQBhruN0HYjSPjC4XkeqW4ny/jNtH9gcbgaqBIXYEZCNnESMAGs0/K/R7oFGMhOyu/eIxg== dependencies: "@babel/helper-plugin-utils" "^7.10.4" - "@babel/helper-regex" "^7.10.4" "@babel/plugin-transform-template-literals@^7.12.1": version "7.12.1" @@ -749,10 +712,10 @@ dependencies: "@babel/helper-plugin-utils" "^7.10.4" -"@babel/plugin-transform-typeof-symbol@^7.12.1": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.12.1.tgz#9ca6be343d42512fbc2e68236a82ae64bc7af78a" - integrity sha512-EPGgpGy+O5Kg5pJFNDKuxt9RdmTgj5sgrus2XVeMp/ZIbOESadgILUbm50SNpghOh3/6yrbsH+NB5+WJTmsA7Q== +"@babel/plugin-transform-typeof-symbol@^7.12.10": + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.12.10.tgz#de01c4c8f96580bd00f183072b0d0ecdcf0dec4b" + integrity sha512-JQ6H8Rnsogh//ijxspCjc21YPd3VLVoYtAwv3zQmqAt8YGYUtdo5usNhdl4b9/Vir2kPFZl6n1h0PfUz4hJhaA== dependencies: "@babel/helper-plugin-utils" "^7.10.4" @@ -781,13 +744,13 @@ "@babel/helper-plugin-utils" "^7.10.4" "@babel/preset-env@^7.0.0": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.12.1.tgz#9c7e5ca82a19efc865384bb4989148d2ee5d7ac2" - integrity sha512-H8kxXmtPaAGT7TyBvSSkoSTUK6RHh61So05SyEbpmr0MCZrsNYn7mGMzzeYoOUCdHzww61k8XBft2TaES+xPLg== + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.12.10.tgz#ca981b95f641f2610531bd71948656306905e6ab" + integrity sha512-Gz9hnBT/tGeTE2DBNDkD7BiWRELZt+8lSysHuDwmYXUIvtwZl0zI+D6mZgXZX0u8YBlLS4tmai9ONNY9tjRgRA== dependencies: - "@babel/compat-data" "^7.12.1" - "@babel/helper-compilation-targets" "^7.12.1" - "@babel/helper-module-imports" "^7.12.1" + "@babel/compat-data" "^7.12.7" + "@babel/helper-compilation-targets" "^7.12.5" + "@babel/helper-module-imports" "^7.12.5" "@babel/helper-plugin-utils" "^7.10.4" "@babel/helper-validator-option" "^7.12.1" "@babel/plugin-proposal-async-generator-functions" "^7.12.1" @@ -797,10 +760,10 @@ "@babel/plugin-proposal-json-strings" "^7.12.1" "@babel/plugin-proposal-logical-assignment-operators" "^7.12.1" "@babel/plugin-proposal-nullish-coalescing-operator" "^7.12.1" - "@babel/plugin-proposal-numeric-separator" "^7.12.1" + "@babel/plugin-proposal-numeric-separator" "^7.12.7" "@babel/plugin-proposal-object-rest-spread" "^7.12.1" "@babel/plugin-proposal-optional-catch-binding" "^7.12.1" - "@babel/plugin-proposal-optional-chaining" "^7.12.1" + "@babel/plugin-proposal-optional-chaining" "^7.12.7" "@babel/plugin-proposal-private-methods" "^7.12.1" "@babel/plugin-proposal-unicode-property-regex" "^7.12.1" "@babel/plugin-syntax-async-generators" "^7.8.0" @@ -842,14 +805,14 @@ "@babel/plugin-transform-reserved-words" "^7.12.1" "@babel/plugin-transform-shorthand-properties" "^7.12.1" "@babel/plugin-transform-spread" "^7.12.1" - "@babel/plugin-transform-sticky-regex" "^7.12.1" + "@babel/plugin-transform-sticky-regex" "^7.12.7" "@babel/plugin-transform-template-literals" "^7.12.1" - "@babel/plugin-transform-typeof-symbol" "^7.12.1" + "@babel/plugin-transform-typeof-symbol" "^7.12.10" "@babel/plugin-transform-unicode-escapes" "^7.12.1" "@babel/plugin-transform-unicode-regex" "^7.12.1" "@babel/preset-modules" "^0.1.3" - "@babel/types" "^7.12.1" - core-js-compat "^3.6.2" + "@babel/types" "^7.12.10" + core-js-compat "^3.8.0" semver "^5.5.0" "@babel/preset-modules@^0.1.3": @@ -864,11 +827,12 @@ esutils "^2.0.2" "@babel/preset-typescript@^7.8.3": - version "7.12.1" - resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.12.1.tgz#86480b483bb97f75036e8864fe404cc782cc311b" - integrity sha512-hNK/DhmoJPsksdHuI/RVrcEws7GN5eamhi28JkO52MqIxU8Z0QpmiSOQxZHWOHV7I3P4UjHV97ay4TcamMA6Kw== + version "7.12.7" + resolved "https://registry.yarnpkg.com/@babel/preset-typescript/-/preset-typescript-7.12.7.tgz#fc7df8199d6aae747896f1e6c61fc872056632a3" + integrity sha512-nOoIqIqBmHBSEgBXWR4Dv/XBehtIFcw9PqZw6rFYuKrzsZmOQm3PR5siLBnKZFEsDb03IegG8nSjU/iXXXYRmw== dependencies: "@babel/helper-plugin-utils" "^7.10.4" + "@babel/helper-validator-option" "^7.12.1" "@babel/plugin-transform-typescript" "^7.12.1" "@babel/runtime@^7.8.4": @@ -878,16 +842,7 @@ dependencies: regenerator-runtime "^0.13.4" -"@babel/template@^7.10.4": - version "7.10.4" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.10.4.tgz#3251996c4200ebc71d1a8fc405fba940f36ba278" - integrity sha512-ZCjD27cGJFUB6nmCB1Enki3r+L5kJveX9pq1SvAUKoICy6CZ9yD8xO086YXdYhvNjBdnekm4ZnaP5yC8Cs/1tA== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/parser" "^7.10.4" - "@babel/types" "^7.10.4" - -"@babel/template@^7.12.7", "@babel/template@^7.3.3": +"@babel/template@^7.10.4", "@babel/template@^7.12.7", "@babel/template@^7.3.3": version "7.12.7" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.12.7.tgz#c817233696018e39fbb6c491d2fb684e05ed43bc" integrity sha512-GkDzmHS6GV7ZeXfJZ0tLRBhZcMcY0/Lnb+eEbXDBfCAcZCjrZKe6p3J4we/D24O9Y8enxWAg1cWwof59yLh2ow== @@ -896,49 +851,25 @@ "@babel/parser" "^7.12.7" "@babel/types" "^7.12.7" -"@babel/traverse@^7.1.0", "@babel/traverse@^7.12.9": - version "7.12.9" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.9.tgz#fad26c972eabbc11350e0b695978de6cc8e8596f" - integrity sha512-iX9ajqnLdoU1s1nHt36JDI9KG4k+vmI8WgjK5d+aDTwQbL2fUnzedNedssA645Ede3PM2ma1n8Q4h2ohwXgMXw== - dependencies: - "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.5" - "@babel/helper-function-name" "^7.10.4" - "@babel/helper-split-export-declaration" "^7.11.0" - "@babel/parser" "^7.12.7" - "@babel/types" "^7.12.7" - debug "^4.1.0" - globals "^11.1.0" - lodash "^4.17.19" - -"@babel/traverse@^7.10.4", "@babel/traverse@^7.12.1", "@babel/traverse@^7.12.5", "@babel/traverse@^7.7.0": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.5.tgz#78a0c68c8e8a35e4cacfd31db8bb303d5606f095" - integrity sha512-xa15FbQnias7z9a62LwYAA5SZZPkHIXpd42C6uW68o8uTuua96FHZy1y61Va5P/i83FAAcMpW8+A/QayntzuqA== +"@babel/traverse@^7.1.0", "@babel/traverse@^7.10.4", "@babel/traverse@^7.12.1", "@babel/traverse@^7.12.10", "@babel/traverse@^7.12.5", "@babel/traverse@^7.7.0": + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.10.tgz#2d1f4041e8bf42ea099e5b2dc48d6a594c00017a" + integrity sha512-6aEtf0IeRgbYWzta29lePeYSk+YAFIC3kyqESeft8o5CkFlYIMX+EQDDWEiAQ9LHOA3d0oHdgrSsID/CKqXJlg== dependencies: "@babel/code-frame" "^7.10.4" - "@babel/generator" "^7.12.5" + "@babel/generator" "^7.12.10" "@babel/helper-function-name" "^7.10.4" "@babel/helper-split-export-declaration" "^7.11.0" - "@babel/parser" "^7.12.5" - "@babel/types" "^7.12.5" + "@babel/parser" "^7.12.10" + "@babel/types" "^7.12.10" debug "^4.1.0" globals "^11.1.0" lodash "^4.17.19" -"@babel/types@^7.0.0", "@babel/types@^7.12.7", "@babel/types@^7.3.0", "@babel/types@^7.3.3": - version "7.12.7" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.7.tgz#6039ff1e242640a29452c9ae572162ec9a8f5d13" - integrity sha512-MNyI92qZq6jrQkXvtIiykvl4WtoRrVV9MPn+ZfsoEENjiWcBQ3ZSHrkxnJWgWtLX3XXqX5hrSQ+X69wkmesXuQ== - dependencies: - "@babel/helper-validator-identifier" "^7.10.4" - lodash "^4.17.19" - to-fast-properties "^2.0.0" - -"@babel/types@^7.10.4", "@babel/types@^7.10.5", "@babel/types@^7.11.0", "@babel/types@^7.12.1", "@babel/types@^7.12.5", "@babel/types@^7.4.4", "@babel/types@^7.7.0": - version "7.12.5" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.5.tgz#5d6b4590cfe90c0c8d7396c83ffd9fc28b5a6450" - integrity sha512-gyTcvz7JFa4V45C0Zklv//GmFOAal5fL23OWpBLqc4nZ4Yrz67s4kCNwSK1Gu0MXGTU8mRY3zJYtacLdKXlzig== +"@babel/types@^7.0.0", "@babel/types@^7.10.4", "@babel/types@^7.10.5", "@babel/types@^7.11.0", "@babel/types@^7.12.1", "@babel/types@^7.12.10", "@babel/types@^7.12.5", "@babel/types@^7.12.7", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0": + version "7.12.10" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.10.tgz#7965e4a7260b26f09c56bcfcb0498af1f6d9b260" + integrity sha512-sf6wboJV5mGyip2hIpDSKsr80RszPinEFjsHTalMxZAZkoQ2/2yQzxlcFN52SJqsyPfLtPmenL4g2KB3KJXPDw== dependencies: "@babel/helper-validator-identifier" "^7.10.4" lodash "^4.17.19" @@ -957,10 +888,10 @@ exec-sh "^0.3.2" minimist "^1.2.0" -"@eslint/eslintrc@^0.2.1": - version "0.2.1" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.2.1.tgz#f72069c330461a06684d119384435e12a5d76e3c" - integrity sha512-XRUeBZ5zBWLYgSANMpThFddrZZkEbGHgUdt5UJjZfnlN9BGCiUBrf+nvbRupSjMvqzwnQN0qwCmOxITt1cfywA== +"@eslint/eslintrc@^0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.2.2.tgz#d01fc791e2fc33e88a29d6f3dc7e93d0cd784b76" + integrity sha512-EfB5OHNYp1F4px/LI/FEnGylop7nOqkQ1LRzCM0KccA2U8tvV8w01KBv37LbO7nW4H+YhKyo2LcJhRwjjV17QQ== dependencies: ajv "^6.12.4" debug "^4.1.1" @@ -1160,12 +1091,22 @@ "@types/yargs" "^15.0.0" chalk "^4.0.0" -"@nicolo-ribaudo/chokidar-2@^2.1.8": - version "2.1.8" - resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8.tgz#eef8d9b47e8dc589499f14d656e8d2dd978c3d14" - integrity sha512-FohwULwAebCUKi/akMFyGi7jfc7JXTeMHzKxuP3umRd9mK/2Y7/SMBSI2jX+YLopPXi+PF9l307NmpfxTdCegA== +"@nicolo-ribaudo/chokidar-2@2.1.8-no-fsevents": + version "2.1.8-no-fsevents" + resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.tgz#da7c3996b8e6e19ebd14d82eaced2313e7769f9b" + integrity sha512-+nb9vWloHNNMFHjGofEam3wopE3m1yuambrrd/fnPc+lFOMB9ROTqQlche9ByFWNkdNqfSgR/kkQtQ8DzEWt2w== dependencies: - chokidar "2.1.8" + anymatch "^2.0.0" + async-each "^1.0.1" + braces "^2.3.2" + glob-parent "^3.1.0" + inherits "^2.0.3" + is-binary-path "^1.0.0" + is-glob "^4.0.0" + normalize-path "^3.0.0" + path-is-absolute "^1.0.0" + readdirp "^2.2.1" + upath "^1.1.1" "@nodelib/fs.scandir@2.1.3": version "2.1.3" @@ -1365,9 +1306,9 @@ "@babel/types" "^7.0.0" "@types/babel__traverse@*", "@types/babel__traverse@^7.0.4", "@types/babel__traverse@^7.0.6": - version "7.0.16" - resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.0.16.tgz#0bbbf70c7bc4193210dd27e252c51260a37cd6a7" - integrity sha512-S63Dt4CZOkuTmpLGGWtT/mQdVORJOpx6SZWGVaP56dda/0Nx5nEe82K7/LAm8zYr6SfMq+1N2OreIOrHAx656w== + version "7.11.0" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.11.0.tgz#b9a1efa635201ba9bc850323a8793ee2d36c04a0" + integrity sha512-kSjgDMZONiIfSH1Nxcr5JIRMwUetDki63FSQfpTCz8ogF3Ulqm8+mr5f78dUYs6vMiB6gBusQqfQmBvHZj/lwg== dependencies: "@babel/types" "^7.3.0" @@ -1384,9 +1325,9 @@ "@types/node" "*" "@types/ioredis@^4.17.7": - version "4.17.7" - resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.17.7.tgz#5007a56380fd387315efabc6c777c050feabb121" - integrity sha512-M8/KDSGmNYhPFMn+CxDDpIEHP27b2muEHgnK1UgIQIMEO2KXH9mznHx3epeRlD+AjF94HFh3dy2/9zqO+pPDqQ== + version "4.17.8" + resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.17.8.tgz#a761c9adac03a9f0413157691fdae247262eb523" + integrity sha512-13WwLG9jMvzjabpBydDXKSPdvAnKI8pZOKk9rEFp3QizyJS8riyNyVRV5ATvU1DCKsz31KM9g90etnTGgMFh3g== dependencies: "@types/node" "*" @@ -1410,9 +1351,9 @@ "@types/istanbul-lib-report" "*" "@types/jest@26.x", "@types/jest@^26.0.15": - version "26.0.15" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.15.tgz#12e02c0372ad0548e07b9f4e19132b834cb1effe" - integrity sha512-s2VMReFXRg9XXxV+CW9e5Nz8fH2K1aEhwgjUqPPbQd7g95T0laAcvLv032EhFHIa5GHsZ8W7iJEQVaJq6k3Gog== + version "26.0.19" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.19.tgz#e6fa1e3def5842ec85045bd5210e9bb8289de790" + integrity sha512-jqHoirTG61fee6v6rwbnEuKhpSKih0tuhqeFbCmMmErhtu3BYlOZaXWjffgOstMM4S/3iQD31lI5bGLTrs97yQ== dependencies: jest-diff "^26.0.0" pretty-format "^26.0.0" @@ -1452,20 +1393,15 @@ dependencies: "@types/node" "*" -"@types/node@*": - version "14.14.5" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.5.tgz#e92d3b8f76583efa26c1a63a21c9d3c1143daa29" - integrity sha512-H5Wn24s/ZOukBmDn03nnGTp18A60ny9AmCwnEcgJiTgSGsCO7k+NWP7zjCCbhlcnVCoI+co52dUAt9GMhOSULw== +"@types/node@*", "@types/node@^14.14.6": + version "14.14.12" + resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.12.tgz#0b1d86f8c40141091285dea02e4940df73bba43f" + integrity sha512-ASH8OPHMNlkdjrEdmoILmzFfsJICvhBsFfAum4aKZ/9U4B6M6tTmTPh+f3ttWdD74CEGV5XvXWkbyfSdXaTd7g== "@types/node@^13.7.0": - version "13.13.34" - resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.34.tgz#c9300a1b6560d90817fb2bba650e250116a575f9" - integrity sha512-g8D1HF2dMDKYSDl5+79izRwRgNPsSynmWMbj50mj7GZ0b7Lv4p8EmZjbo3h0h+6iLr6YmVz9VnF6XVZ3O6V1Ug== - -"@types/node@^14.14.6": - version "14.14.6" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.6.tgz#146d3da57b3c636cc0d1769396ce1cfa8991147f" - integrity sha512-6QlRuqsQ/Ox/aJEQWBEJG7A9+u7oSYl3mem/K8IzxXG/kAGbV1YPD9Bg9Zw3vyxC/YP+zONKwy8hGkSt1jxFMw== + version "13.13.35" + resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.35.tgz#d417b48313d691f5c8ff9c52cbc19cdecd306b5e" + integrity sha512-q9aeOGwv+RRou/ca4aJVUM/jD5u7LBexu+rq9PkA/NhHNn8JifcMo94soKm0b6JGSfw/PSNdqtc428OscMvEYA== "@types/normalize-package-data@^2.4.0": version "2.4.0" @@ -1483,9 +1419,9 @@ integrity sha512-L8ogeT6vDzT1vxlW3KITTCt+BVXXVkLXfZ/XNm6UqbcJgxf+KPO7yjWx7dQQE8RW07KopL10x2gNMs41+IkMGQ== "@types/pg@^7.14.6": - version "7.14.6" - resolved "https://registry.yarnpkg.com/@types/pg/-/pg-7.14.6.tgz#10d2ab6ae8bdda38a77ef677f15d24e9a66d526e" - integrity sha512-mf6kYvuYS2CObDxtsw5imE4vOvq+XHJtiJNFXir/fn9nxZtIFQ3/ECpokm8fNkxKBh7nQhyegHwNg1NIW+yYJA== + version "7.14.7" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-7.14.7.tgz#b25532a424f58e70432ac31c77507dfb7b9349a8" + integrity sha512-ZnMOUidTP6Lwsb0bxHL6PVIL1lVC2CYNQWlA79kQ6nn0rK1/ynvkyN1wsR9pVZaP4WcCNioKT/2aU5UuLIQy2w== dependencies: "@types/node" "*" "@types/pg-types" "*" @@ -1542,75 +1478,68 @@ resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d" integrity sha512-FA/BWv8t8ZWJ+gEOnLLd8ygxH/2UFbAvgEonyfN6yWGLKc7zVjbpl2Y4CTjid9h2RfgPP6SEt6uHwEOply00yw== -"@types/yargs@^15.0.0": - version "15.0.10" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.10.tgz#0fe3c8173a0d5c3e780b389050140c3f5ea6ea74" - integrity sha512-z8PNtlhrj7eJNLmrAivM7rjBESG6JwC5xP3RVk12i/8HVP7Xnx/sEmERnRImyEuUaJfO942X0qMOYsoupaJbZQ== - dependencies: - "@types/yargs-parser" "*" - -"@types/yargs@^15.0.9": - version "15.0.9" - resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.9.tgz#524cd7998fe810cdb02f26101b699cccd156ff19" - integrity sha512-HmU8SeIRhZCWcnRskCs36Q1Q00KBV6Cqh/ora8WN1+22dY07AZdn6Gel8QZ3t26XYPImtcL8WV/eqjhVmMEw4g== +"@types/yargs@^15.0.0", "@types/yargs@^15.0.9": + version "15.0.11" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.11.tgz#361d7579ecdac1527687bcebf9946621c12ab78c" + integrity sha512-jfcNBxHFYJ4nPIacsi3woz1+kvUO6s1CyeEhtnDHBjHUMNj5UlW2GynmnSgiJJEdNg9yW5C8lfoNRZrHGv5EqA== dependencies: "@types/yargs-parser" "*" "@typescript-eslint/eslint-plugin@^4.6.1": - version "4.6.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.6.1.tgz#99d77eb7a016fd5a5e749d2c44a7e4c317eb7da3" - integrity sha512-SNZyflefTMK2JyrPfFFzzoy2asLmZvZJ6+/L5cIqg4HfKGiW2Gr1Go1OyEVqne/U4QwmoasuMwppoBHWBWF2nA== + version "4.9.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.9.1.tgz#66758cbe129b965fe9c63b04b405d0cf5280868b" + integrity sha512-QRLDSvIPeI1pz5tVuurD+cStNR4sle4avtHhxA+2uyixWGFjKzJ+EaFVRW6dA/jOgjV5DTAjOxboQkRDE8cRlQ== dependencies: - "@typescript-eslint/experimental-utils" "4.6.1" - "@typescript-eslint/scope-manager" "4.6.1" + "@typescript-eslint/experimental-utils" "4.9.1" + "@typescript-eslint/scope-manager" "4.9.1" debug "^4.1.1" functional-red-black-tree "^1.0.1" regexpp "^3.0.0" semver "^7.3.2" tsutils "^3.17.1" -"@typescript-eslint/experimental-utils@4.6.1": - version "4.6.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.6.1.tgz#a9c691dfd530a9570274fe68907c24c07a06c4aa" - integrity sha512-qyPqCFWlHZXkEBoV56UxHSoXW2qnTr4JrWVXOh3soBP3q0o7p4pUEMfInDwIa0dB/ypdtm7gLOS0hg0a73ijfg== +"@typescript-eslint/experimental-utils@4.9.1": + version "4.9.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.9.1.tgz#86633e8395191d65786a808dc3df030a55267ae2" + integrity sha512-c3k/xJqk0exLFs+cWSJxIjqLYwdHCuLWhnpnikmPQD2+NGAx9KjLYlBDcSI81EArh9FDYSL6dslAUSwILeWOxg== dependencies: "@types/json-schema" "^7.0.3" - "@typescript-eslint/scope-manager" "4.6.1" - "@typescript-eslint/types" "4.6.1" - "@typescript-eslint/typescript-estree" "4.6.1" + "@typescript-eslint/scope-manager" "4.9.1" + "@typescript-eslint/types" "4.9.1" + "@typescript-eslint/typescript-estree" "4.9.1" eslint-scope "^5.0.0" eslint-utils "^2.0.0" "@typescript-eslint/parser@^4.6.1": - version "4.6.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.6.1.tgz#b801bff67b536ecc4a840ac9289ba2be57e02428" - integrity sha512-lScKRPt1wM9UwyKkGKyQDqf0bh6jm8DQ5iN37urRIXDm16GEv+HGEmum2Fc423xlk5NUOkOpfTnKZc/tqKZkDQ== + version "4.9.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.9.1.tgz#2d74c4db5dd5117379a9659081a4d1ec02629055" + integrity sha512-Gv2VpqiomvQ2v4UL+dXlQcZ8zCX4eTkoIW+1aGVWT6yTO+6jbxsw7yQl2z2pPl/4B9qa5JXeIbhJpONKjXIy3g== dependencies: - "@typescript-eslint/scope-manager" "4.6.1" - "@typescript-eslint/types" "4.6.1" - "@typescript-eslint/typescript-estree" "4.6.1" + "@typescript-eslint/scope-manager" "4.9.1" + "@typescript-eslint/types" "4.9.1" + "@typescript-eslint/typescript-estree" "4.9.1" debug "^4.1.1" -"@typescript-eslint/scope-manager@4.6.1": - version "4.6.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.6.1.tgz#21872b91cbf7adfc7083f17b8041149148baf992" - integrity sha512-f95+80r6VdINYscJY1KDUEDcxZ3prAWHulL4qRDfNVD0I5QAVSGqFkwHERDoLYJJWmEAkUMdQVvx7/c2Hp+Bjg== +"@typescript-eslint/scope-manager@4.9.1": + version "4.9.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.9.1.tgz#cc2fde310b3f3deafe8436a924e784eaab265103" + integrity sha512-sa4L9yUfD/1sg9Kl8OxPxvpUcqxKXRjBeZxBuZSSV1v13hjfEJkn84n0An2hN8oLQ1PmEl2uA6FkI07idXeFgQ== dependencies: - "@typescript-eslint/types" "4.6.1" - "@typescript-eslint/visitor-keys" "4.6.1" + "@typescript-eslint/types" "4.9.1" + "@typescript-eslint/visitor-keys" "4.9.1" -"@typescript-eslint/types@4.6.1": - version "4.6.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.6.1.tgz#d3ad7478f53f22e7339dc006ab61aac131231552" - integrity sha512-k2ZCHhJ96YZyPIsykickez+OMHkz06xppVLfJ+DY90i532/Cx2Z+HiRMH8YZQo7a4zVd/TwNBuRCdXlGK4yo8w== +"@typescript-eslint/types@4.9.1": + version "4.9.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.9.1.tgz#a1a7dd80e4e5ac2c593bc458d75dd1edaf77faa2" + integrity sha512-fjkT+tXR13ks6Le7JiEdagnwEFc49IkOyys7ueWQ4O8k4quKPwPJudrwlVOJCUQhXo45PrfIvIarcrEjFTNwUA== -"@typescript-eslint/typescript-estree@4.6.1": - version "4.6.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.6.1.tgz#6025cce724329413f57e4959b2d676fceeca246f" - integrity sha512-/J/kxiyjQQKqEr5kuKLNQ1Finpfb8gf/NpbwqFFYEBjxOsZ621r9AqwS9UDRA1Rrr/eneX/YsbPAIhU2rFLjXQ== +"@typescript-eslint/typescript-estree@4.9.1": + version "4.9.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.9.1.tgz#6e5b86ff5a5f66809e1f347469fadeec69ac50bf" + integrity sha512-bzP8vqwX6Vgmvs81bPtCkLtM/Skh36NE6unu6tsDeU/ZFoYthlTXbBmpIrvosgiDKlWTfb2ZpPELHH89aQjeQw== dependencies: - "@typescript-eslint/types" "4.6.1" - "@typescript-eslint/visitor-keys" "4.6.1" + "@typescript-eslint/types" "4.9.1" + "@typescript-eslint/visitor-keys" "4.9.1" debug "^4.1.1" globby "^11.0.1" is-glob "^4.0.1" @@ -1618,12 +1547,12 @@ semver "^7.3.2" tsutils "^3.17.1" -"@typescript-eslint/visitor-keys@4.6.1": - version "4.6.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.6.1.tgz#6b125883402d8939df7b54528d879e88f7ba3614" - integrity sha512-owABze4toX7QXwOLT3/D5a8NecZEjEWU1srqxENTfqsY3bwVnl3YYbOh6s1rp2wQKO9RTHFGjKes08FgE7SVMw== +"@typescript-eslint/visitor-keys@4.9.1": + version "4.9.1" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.9.1.tgz#d76374a58c4ead9e92b454d186fea63487b25ae1" + integrity sha512-9gspzc6UqLQHd7lXQS7oWs+hrYggspv/rk6zzEMhCbYwPE/sF7oxo7GAjkS35Tdlt7wguIG+ViWCPtVZHz/ybQ== dependencies: - "@typescript-eslint/types" "4.6.1" + "@typescript-eslint/types" "4.9.1" eslint-visitor-keys "^2.0.0" abab@^2.0.3: @@ -1644,7 +1573,7 @@ acorn-globals@^6.0.0: acorn "^7.1.1" acorn-walk "^7.1.1" -acorn-jsx@^5.2.0: +acorn-jsx@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b" integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng== @@ -1784,12 +1713,14 @@ array-from@^2.1.1: integrity sha1-z+nYwmYoudxa7MYqn12PHzUsEZU= array-includes@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.1.tgz#cdd67e6852bdf9c1215460786732255ed2459348" - integrity sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ== + version "3.1.2" + resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.2.tgz#a8db03e0b88c8c6aeddc49cb132f9bcab4ebf9c8" + integrity sha512-w2GspexNQpx+PutG3QpT437/BenZBj0M/MZGn5mzv/MofYqo0xmRHzn4lFsoDlWJ+THYsGJmFlW68WlDFx7VRw== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.17.0" + es-abstract "^1.18.0-next.1" + get-intrinsic "^1.0.1" is-string "^1.0.5" array-union@^2.1.0: @@ -1803,20 +1734,22 @@ array-unique@^0.3.2: integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= array.prototype.flat@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz#0de82b426b0318dbfdb940089e38b043d37f6c7b" - integrity sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ== + version "1.2.4" + resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.4.tgz#6ef638b43312bd401b4c6199fdec7e2dc9e9a123" + integrity sha512-4470Xi3GAPAjZqFcljX2xzckv1qeKPizoNkiS0+O4IoPR2ZNpcjE0pkhdihlDouK+x6QOast26B4Q/O9DJnwSg== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.17.0-next.1" + es-abstract "^1.18.0-next.1" array.prototype.flatmap@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.2.3.tgz#1c13f84a178566042dd63de4414440db9222e443" - integrity sha512-OOEk+lkePcg+ODXIpvuU9PAryCikCJyo7GlDG1upleEpQRx6mzL9puEBkozQ5iAx20KV0l3DbyQwqciJtqe5Pg== + version "1.2.4" + resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.2.4.tgz#94cfd47cc1556ec0747d97f7c7738c58122004c9" + integrity sha512-r9Z0zYoxqHz60vvQbWEdXIEtCwHF0yxaWfno9qzXeNHvfyl3BZqygmGzb84dsubyaXLH4husF+NFgMSdpZhk2Q== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.17.0-next.1" + es-abstract "^1.18.0-next.1" function-bind "^1.1.1" asn1@~0.2.3: @@ -1976,16 +1909,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= -base64-js@^1.2.0: +base64-js@^1.2.0, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -base64-js@^1.3.1: - version "1.3.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" - integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== - base@^0.11.1: version "0.11.2" resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" @@ -2016,7 +1944,7 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== -bindings@^1.3.0, bindings@^1.3.1, bindings@^1.5.0: +bindings@^1.3.0, bindings@^1.3.1: version "1.5.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== @@ -2068,15 +1996,16 @@ browser-process-hrtime@^1.0.0: resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== -browserslist@^4.14.5, browserslist@^4.8.5: - version "4.14.6" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.14.6.tgz#97702a9c212e0c6b6afefad913d3a1538e348457" - integrity sha512-zeFYcUo85ENhc/zxHbiIp0LGzzTrE2Pv2JhxvS7kpUb9Q9D38kUX6Bie7pGutJ/5iF5rOxE7CepAuWD56xJ33A== +browserslist@^4.14.5, browserslist@^4.15.0: + version "4.16.0" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.0.tgz#410277627500be3cb28a1bfe037586fbedf9488b" + integrity sha512-/j6k8R0p3nxOC6kx5JGAxsnhc9ixaWJfYc+TNTzxg6+ARaESAvQGV7h0uNOB4t+pLQJZWzcrMxXOxjgsCj3dqQ== dependencies: - caniuse-lite "^1.0.30001154" - electron-to-chromium "^1.3.585" + caniuse-lite "^1.0.30001165" + colorette "^1.2.1" + electron-to-chromium "^1.3.621" escalade "^3.1.1" - node-releases "^1.1.65" + node-releases "^1.1.67" bs-logger@0.x: version "0.2.6" @@ -2103,9 +2032,9 @@ buffer-writer@2.0.0: integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw== buffer@^5.5.0: - version "5.7.0" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.0.tgz#88afbd29fc89fa7b58e82b39206f31f2cf34feed" - integrity sha512-cd+5r1VLBwUqTrmnzW+D7ABkJUM6mr7uv1dv+6jRw4Rcl7tFIFHDqHPL98LhpGFn3dbAt3gtLxtrWp4m1kFrqg== + version "5.7.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" + integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== dependencies: base64-js "^1.3.1" ieee754 "^1.1.13" @@ -2161,10 +2090,10 @@ camelcase@^6.0.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809" integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg== -caniuse-lite@^1.0.30001154: - version "1.0.30001154" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001154.tgz#f3bbc245ce55e4c1cd20fa731b097880181a7f17" - integrity sha512-y9DvdSti8NnYB9Be92ddMZQrcOe04kcQtcxtBx4NkB04+qZ+JUWotnXBJTmxlKudhxNTQ3RRknMwNU2YQl/Org== +caniuse-lite@^1.0.30001165: + version "1.0.30001165" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001165.tgz#32955490d2f60290bb186bb754f2981917fa744f" + integrity sha512-8cEsSMwXfx7lWSUMA2s08z9dIgsnR5NAqjXP23stdsU3AUWkCr/rr4s4OFtHXn5XXr6+7kam3QFVoYyXNPdJPA== capture-exit@^2.0.0: version "2.0.0" @@ -2200,25 +2129,6 @@ char-regex@^1.0.2: resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== -chokidar@2.1.8: - version "2.1.8" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" - integrity sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg== - dependencies: - anymatch "^2.0.0" - async-each "^1.0.1" - braces "^2.3.2" - glob-parent "^3.1.0" - inherits "^2.0.3" - is-binary-path "^1.0.0" - is-glob "^4.0.0" - normalize-path "^3.0.0" - path-is-absolute "^1.0.0" - readdirp "^2.2.1" - upath "^1.1.1" - optionalDependencies: - fsevents "^1.2.7" - chokidar@^3.4.0: version "3.4.3" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.4.3.tgz#c1df38231448e45ca4ac588e6c79573ba6a57d5b" @@ -2284,9 +2194,9 @@ cliui@^6.0.0: wrap-ansi "^6.2.0" cliui@^7.0.2: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.3.tgz#ef180f26c8d9bff3927ee52428bfec2090427981" - integrity sha512-Gj3QHTkVMPKqwP3f7B4KPkBZRMR9r4rfi5bXFpg1a+Svvj8l7q5CnkBkVQzfxT5DFSsGk2+PascOgL0JYkL2kw== + version "7.0.4" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f" + integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ== dependencies: string-width "^4.2.0" strip-ansi "^6.0.0" @@ -2339,6 +2249,11 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +colorette@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" + integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== + combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -2398,12 +2313,12 @@ copy-descriptor@^0.1.0: resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= -core-js-compat@^3.6.2: - version "3.6.5" - resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.6.5.tgz#2a51d9a4e25dfd6e690251aa81f99e3c05481f1c" - integrity sha512-7ItTKOhOZbznhXAQ2g/slGg1PJV5zDO/WdkTwi7UEOJmkvsE32PWvx6mKtDjiMpjnR2CNf6BAD6sSxIlv7ptng== +core-js-compat@^3.8.0: + version "3.8.1" + resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.8.1.tgz#8d1ddd341d660ba6194cbe0ce60f4c794c87a36e" + integrity sha512-a16TLmy9NVD1rkjUGbwuyWkiDoN0FDpAwrfLONvHFQx0D9k7J9y0srwMT8QP/Z6HE3MIFaVynEeYwZwPX1o5RQ== dependencies: - browserslist "^4.8.5" + browserslist "^4.15.0" semver "7.0.0" core-util-is@1.0.2, core-util-is@~1.0.0: @@ -2422,10 +2337,15 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +create-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" + integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== + cron-parser@^2.7.3: - version "2.16.3" - resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.16.3.tgz#acb8e405eed1733aac542fdf604cb7c1daf0204a" - integrity sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg== + version "2.18.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.18.0.tgz#de1bb0ad528c815548371993f81a54e5a089edcf" + integrity sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg== dependencies: is-nan "^1.3.0" moment-timezone "^0.5.31" @@ -2506,7 +2426,7 @@ dateformat@~1.0.4-1.2.3: get-stdin "^4.0.1" meow "^3.3.0" -debug@4, debug@^4.0.0: +debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== @@ -2520,13 +2440,6 @@ debug@^2.2.0, debug@^2.3.3, debug@^2.6.9: dependencies: ms "2.0.0" -debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1" - integrity sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg== - dependencies: - ms "2.1.2" - decamelize@^1.1.2, decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -2662,10 +2575,10 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" -electron-to-chromium@^1.3.585: - version "1.3.587" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.587.tgz#de570df7320eb259c0511f284c2d6008094edbf7" - integrity sha512-8XFNxzNj0R8HpTQslWAw6UWpGSuOKSP3srhyFHVbGUGb8vTHckZGCyWi+iQlaXJx5DNeTQTQLd6xN11WSckkmA== +electron-to-chromium@^1.3.621: + version "1.3.622" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.622.tgz#9726bd2e67a5462154750ce9701ca6af07d07877" + integrity sha512-AJT0Fm1W0uZlMVVkkJrcCVvczDuF8tPm3bwzQf5WO8AaASB2hwTRP7B8pU5rqjireH+ib6am8+hH5/QkXzzYKw== emittery@^0.7.1: version "0.7.2" @@ -2703,7 +2616,7 @@ error-ex@^1.2.0, error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.17.0, es-abstract@^1.17.0-next.1, es-abstract@^1.17.5: +es-abstract@^1.17.0-next.1: version "1.17.7" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.17.7.tgz#a4de61b2f66989fc7421676c1cb9787573ace54c" integrity sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g== @@ -2894,9 +2807,9 @@ eslint-plugin-node@^11.1.0: semver "^6.1.0" eslint-plugin-prettier@^3.1.2: - version "3.1.4" - resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.4.tgz#168ab43154e2ea57db992a2cd097c828171f75c2" - integrity sha512-jZDa8z76klRqo+TdGDTFJSavwbnWK2ZpqGKNZ+VvweMW516pDUMmQ2koXvxEE4JhzNvTv+radye/bWGBmA6jmg== + version "3.2.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-3.2.0.tgz#af391b2226fa0e15c96f36c733f6e9035dbd952c" + integrity sha512-kOUSJnFjAUFKwVxuzy6sA5yyMx6+o9ino4gCdShzBNx4eyFRudWRYKCFolKjoM40PEiuU6Cn7wBLfq3WsGg7qg== dependencies: prettier-linter-helpers "^1.0.0" @@ -2923,9 +2836,9 @@ eslint-plugin-react@^7.20.0: string.prototype.matchall "^4.0.2" eslint-plugin-standard@^4.0.0: - version "4.0.2" - resolved "https://registry.yarnpkg.com/eslint-plugin-standard/-/eslint-plugin-standard-4.0.2.tgz#021211a9f077e63a6847e7bb9ab4247327ac8e0c" - integrity sha512-nKptN8l7jksXkwFk++PhJB3cCDTcXOEyhISIN86Ue2feJ1LFyY3PrY3/xT2keXlJSY5bpmbiTG0f885/YKAvTA== + version "4.1.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz#0c3bf3a67e853f8bbbc580fb4945fbf16f41b7c5" + integrity sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ== eslint-scope@^5.0.0, eslint-scope@^5.1.1: version "5.1.1" @@ -2953,12 +2866,12 @@ eslint-visitor-keys@^2.0.0: integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== eslint@^7.0.0: - version "7.12.1" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.12.1.tgz#bd9a81fa67a6cfd51656cdb88812ce49ccec5801" - integrity sha512-HlMTEdr/LicJfN08LB3nM1rRYliDXOmfoO4vj39xN6BLpFzF00hbwBoqHk8UcJ2M/3nlARZWy/mslvGEuZFvsg== + version "7.15.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.15.0.tgz#eb155fb8ed0865fcf5d903f76be2e5b6cd7e0bc7" + integrity sha512-Vr64xFDT8w30wFll643e7cGrIkPEU50yIiI36OdSIDoSGguIeaLzBo0vpGvzo9RECUqq7htURfwEtKqwytkqzA== dependencies: "@babel/code-frame" "^7.0.0" - "@eslint/eslintrc" "^0.2.1" + "@eslint/eslintrc" "^0.2.2" ajv "^6.10.0" chalk "^4.0.0" cross-spawn "^7.0.2" @@ -2968,10 +2881,10 @@ eslint@^7.0.0: eslint-scope "^5.1.1" eslint-utils "^2.1.0" eslint-visitor-keys "^2.0.0" - espree "^7.3.0" + espree "^7.3.1" esquery "^1.2.0" esutils "^2.0.2" - file-entry-cache "^5.0.1" + file-entry-cache "^6.0.0" functional-red-black-tree "^1.0.1" glob-parent "^5.0.0" globals "^12.1.0" @@ -2995,13 +2908,13 @@ eslint@^7.0.0: text-table "^0.2.0" v8-compile-cache "^2.0.3" -espree@^7.3.0: - version "7.3.0" - resolved "https://registry.yarnpkg.com/espree/-/espree-7.3.0.tgz#dc30437cf67947cf576121ebd780f15eeac72348" - integrity sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw== +espree@^7.3.0, espree@^7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/espree/-/espree-7.3.1.tgz#f2df330b752c6f55019f8bd89b7660039c1bbbb6" + integrity sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g== dependencies: acorn "^7.4.0" - acorn-jsx "^5.2.0" + acorn-jsx "^5.3.1" eslint-visitor-keys "^1.3.0" esprima@^4.0.0, esprima@^4.0.1: @@ -3232,9 +3145,9 @@ fastify-warning@^0.2.0: integrity sha512-s1EQguBw/9qtc1p/WTY4eq9WMRIACkj+HTcOIK1in4MV5aFaQC9ZCIt0dJ7pr5bIf4lPpHvAtP2ywpTNgs7hqw== fastify@^3.8.0: - version "3.8.0" - resolved "https://registry.yarnpkg.com/fastify/-/fastify-3.8.0.tgz#455bfa70394322247050c330d0e52532b349662d" - integrity sha512-w57/uvyQWzF/KSr9CbWQ5nfTqSSfYcmrems9Lc3VvtrAF7EsLbfZQBQZul6xwvE1uEfxA4nGdoUKqpU7xiv7cw== + version "3.9.1" + resolved "https://registry.yarnpkg.com/fastify/-/fastify-3.9.1.tgz#fe7faba92c72fb90054c5f3422965e34142f4cdf" + integrity sha512-J+JyYc2PH7N6KxrG0d3d2V0HRPdBMpxp3eKdGcDtemTvXhuQKFHFLQ+WkeFLShkcGDsf+/vBV+NMK8Ki97/OTg== dependencies: abstract-logging "^2.0.0" ajv "^6.12.2" @@ -3288,12 +3201,12 @@ figures@^3.2.0: dependencies: escape-string-regexp "^1.0.5" -file-entry-cache@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-5.0.1.tgz#ca0f6efa6dd3d561333fb14515065c2fafdf439c" - integrity sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g== +file-entry-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.0.tgz#7921a89c391c6d93efec2169ac6bf300c527ea0a" + integrity sha512-fqoO76jZ3ZnYrXLDRxBR1YvOvc0k844kcOg40bgsPrE25LAb/PDqTY+ho64Xh2c8ZXgIKldchCFHczG2UVRcWA== dependencies: - flat-cache "^2.0.1" + flat-cache "^3.0.4" file-uri-to-path@1.0.0: version "1.0.0" @@ -3356,24 +3269,23 @@ find-versions@^3.2.0: dependencies: semver-regex "^2.0.0" -flat-cache@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-2.0.1.tgz#5d296d6f04bda44a4630a301413bdbc2ec085ec0" - integrity sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA== +flat-cache@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11" + integrity sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg== dependencies: - flatted "^2.0.0" - rimraf "2.6.3" - write "1.0.3" + flatted "^3.1.0" + rimraf "^3.0.2" flatstr@^1.0.12: version "1.0.12" resolved "https://registry.yarnpkg.com/flatstr/-/flatstr-1.0.12.tgz#c2ba6a08173edbb6c9640e3055b95e287ceb5931" integrity sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw== -flatted@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138" - integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA== +flatted@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.0.tgz#a5d06b4a8b01e3a63771daa5cb7a1903e2e57067" + integrity sha512-tW+UkmtNg/jv9CSofAKvgVcO7c2URjhTdW1ZTkcAritblu8tajiYy7YisnIflEwtKssCtOxpnBRoCB7iap0/TA== for-in@^1.0.2: version "1.0.2" @@ -3435,14 +3347,6 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= -fsevents@^1.2.7: - version "1.2.13" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.2.13.tgz#f325cb0455592428bcf11b383370ef70e3bfcc38" - integrity sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw== - dependencies: - bindings "^1.5.0" - nan "^2.12.1" - fsevents@^2.1.2: version "2.2.1" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.2.1.tgz#1fb02ded2036a8ac288d507a65962bd87b97628d" @@ -3473,7 +3377,7 @@ get-caller-file@^2.0.1, get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.0: +get-intrinsic@^1.0.0, get-intrinsic@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.0.1.tgz#94a9768fcbdd0595a1c9273aacf4c89d075631be" integrity sha512-ZnWP+AmS1VUaLgTRy47+zKtjTxz+0xMpx3I52i+aalBK1QP19ggLF3Db89KJX7kjfOfP2eoa01qc++GwPgufPg== @@ -3716,9 +3620,9 @@ human-signals@^1.1.1: integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw== husky@>=4: - version "4.3.0" - resolved "https://registry.yarnpkg.com/husky/-/husky-4.3.0.tgz#0b2ec1d66424e9219d359e26a51c58ec5278f0de" - integrity sha512-tTMeLCLqSBqnflBZnlVDhpaIMucSGaYyX6855jM4AguGeWCeSzNdb1mfyWduTZ3pe3SJVvVWGL0jO1iKZVPfTA== + version "4.3.5" + resolved "https://registry.yarnpkg.com/husky/-/husky-4.3.5.tgz#ab8d2a0eb6b62fef2853ee3d442c927d89290902" + integrity sha512-E5S/1HMoDDaqsH8kDF5zeKEQbYqe3wL9zJDyqyYqc8I4vHBtAoxkDBGXox0lZ9RI+k5GyB728vZdmnM4bYap+g== dependencies: chalk "^4.0.0" ci-info "^2.0.0" @@ -3753,7 +3657,7 @@ ignore@^5.1.1, ignore@^5.1.4: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== -import-fresh@^3.0.0: +import-fresh@^3.0.0, import-fresh@^3.2.1: version "3.2.2" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.2.2.tgz#fc129c160c5d68235507f4331a6baad186bdbc3e" integrity sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw== @@ -3761,14 +3665,6 @@ import-fresh@^3.0.0: parent-module "^1.0.0" resolve-from "^4.0.0" -import-fresh@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.2.1.tgz#633ff618506e793af5ac91bf48b72677e15cbe66" - integrity sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ== - dependencies: - parent-module "^1.0.0" - resolve-from "^4.0.0" - import-local@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.0.2.tgz#a8cfd0431d1de4a2199703d003e3e62364fa6db6" @@ -3906,17 +3802,10 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-core-module@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.0.0.tgz#58531b70aed1db7c0e8d4eb1a0a2d1ddd64bd12d" - integrity sha512-jq1AH6C8MuteOoBPwkxHafmByhL9j5q4OaPGdbuD+ZtQJVzH+i6E3BJDQcBA09k57i2Hh2yQbEG8yObZ0jdlWw== - dependencies: - has "^1.0.3" - is-core-module@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946" - integrity sha512-YcV7BgVMRFRua2FqQzKtTDMz8iCuLEyGKjr70q8Zm1yy2qKcurbFEd79PAdHV77oL3NrAaOVQIbMmiHQCHB7ZA== + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.2.0.tgz#97037ef3d52224d85163f5597b2b63d9afed981a" + integrity sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ== dependencies: has "^1.0.3" @@ -4021,9 +3910,9 @@ is-nan@^1.3.0: define-properties "^1.1.3" is-negative-zero@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.0.tgz#9553b121b0fac28869da9ed459e20c7543788461" - integrity sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE= + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" + integrity sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w== is-number@^3.0.0: version "3.0.0" @@ -4557,9 +4446,9 @@ jest@^26.6.3: integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== js-yaml@^3.13.1: - version "3.14.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.0.tgz#a7a34170f26a21bb162424d8adacb4113a69e482" - integrity sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A== + version "3.14.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== dependencies: argparse "^1.0.7" esprima "^4.0.0" @@ -4735,9 +4624,9 @@ lines-and-columns@^1.1.6: integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= lint-staged@>=10.5.1: - version "10.5.1" - resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-10.5.1.tgz#901e915c2360072dded0e7d752a0d9a49e079daa" - integrity sha512-fTkTGFtwFIJJzn/PbUO3RXyEBHIhbfYBE7+rJyLcOXabViaO/h6OslgeK6zpeUtzkDrzkgyAYDTLAwx6JzDTHw== + version "10.5.3" + resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-10.5.3.tgz#c682838b3eadd4c864d1022da05daa0912fb1da5" + integrity sha512-TanwFfuqUBLufxCc3RUtFEkFraSPNR3WzWcGF39R3f2J7S9+iF9W0KTVLfSy09lYGmZS5NDCxjNvhGMSJyFCWg== dependencies: chalk "^4.1.0" cli-truncate "^2.1.0" @@ -4756,9 +4645,9 @@ lint-staged@>=10.5.1: stringify-object "^3.3.0" listr2@^3.2.2: - version "3.2.2" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.2.2.tgz#d20feb75015e506992b55af40722ba1af168b8f1" - integrity sha512-AajqcZEUikF2ioph6PfH3dIuxJclhr3i3kHgTOP0xeXdWQohrvJAAmqVcV43/GI987HFY/vzT73jYXoa4esDHg== + version "3.2.3" + resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.2.3.tgz#ef9e0d790862f038dde8a9837be552b1adfd1c07" + integrity sha512-vUb80S2dSUi8YxXahO8/I/s29GqnOL8ozgHVLjfWQXa03BNEeS1TpBLjh2ruaqq5ufx46BRGvfymdBSuoXET5w== dependencies: chalk "^4.1.0" cli-truncate "^2.1.0" @@ -4872,6 +4761,13 @@ loud-rejection@^1.0.0: currently-unhandled "^0.4.1" signal-exit "^3.0.0" +lru-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + lru_map@^0.3.3: version "0.3.3" resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd" @@ -5031,17 +4927,10 @@ mkdirp@1.x, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -mkdirp@^0.5.1: - version "0.5.5" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" - integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== - dependencies: - minimist "^1.2.5" - moment-timezone@^0.5.31: - version "0.5.31" - resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.31.tgz#9c40d8c5026f0c7ab46eda3d63e49c155148de05" - integrity sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA== + version "0.5.32" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.32.tgz#db7677cc3cc680fd30303ebd90b0da1ca0dfecc2" + integrity sha512-Z8QNyuQHQAmWucp8Knmgei8YNo28aLjJq6Ma+jy1ZSpSk5nyfRT8xgUbSQvD2+2UajISfenndwvFuH3NGS+nvA== dependencies: moment ">= 2.9.0" @@ -5060,7 +4949,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -nan@^2.12.1, nan@^2.13.2, nan@^2.14.0: +nan@^2.13.2, nan@^2.14.0: version "2.14.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== @@ -5142,10 +5031,10 @@ node-notifier@^8.0.0: uuid "^8.3.0" which "^2.0.2" -node-releases@^1.1.65: - version "1.1.65" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.65.tgz#52d9579176bd60f23eba05c4438583f341944b81" - integrity sha512-YpzJOe2WFIW0V4ZkJQd/DGR/zdVwc/pI4Nl1CZrBO19FdRcSTmsuhdttw9rsTzzJLrNcSloLiBbEYx1C4f6gpA== +node-releases@^1.1.67: + version "1.1.67" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.67.tgz#28ebfcccd0baa6aad8e8d4d8fe4cbc49ae239c12" + integrity sha512-V5QF9noGFl3EymEwUYzO+3NTDpGfQB4ve6Qfnzf3UNydMhjQRVPR1DZTuvWiLzaFJYw2fmDwAfnRNEVb64hSIg== node-schedule@^1.3.2: version "1.3.2" @@ -5217,9 +5106,9 @@ object-copy@^0.1.0: kind-of "^3.0.3" object-inspect@^1.8.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0" - integrity sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA== + version "1.9.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.9.0.tgz#c90521d74e1127b67266ded3394ad6116986533a" + integrity sha512-i3Bp9iTqwhaLZBxGkRfo5ZbE07BQRT7MGu8+nNgwW9ItGp1TzCTw2DLEoWwjClxBjOFI/hWljTAmYGCEwmtnOw== object-keys@^1.0.12, object-keys@^1.1.1: version "1.1.1" @@ -5244,22 +5133,23 @@ object.assign@^4.1.0, object.assign@^4.1.1: object-keys "^1.1.1" object.entries@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.2.tgz#bc73f00acb6b6bb16c203434b10f9a7e797d3add" - integrity sha512-BQdB9qKmb/HyNdMNWVr7O3+z5MUIx3aiegEIJqjMBbBf0YT9RRxTJSim4mzFqtyr7PDAHigq0N9dO0m0tRakQA== + version "1.1.3" + resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.3.tgz#c601c7f168b62374541a07ddbd3e2d5e4f7711a6" + integrity sha512-ym7h7OZebNS96hn5IJeyUmaWhaSM4SVtAPPfNLQEI2MYWCO2egsITb9nab2+i/Pwibx+R0mtn+ltKJXRSeTMGg== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.17.5" + es-abstract "^1.18.0-next.1" has "^1.0.3" object.fromentries@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.2.tgz#4a09c9b9bb3843dd0f89acdb517a794d4f355ac9" - integrity sha512-r3ZiBH7MQppDJVLx6fhD618GKNG40CZYH9wgwdhKxBDDbQgjeWGGd4AtkZad84d291YxvWe7bJGuE65Anh0dxQ== + version "2.0.3" + resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.3.tgz#13cefcffa702dc67750314a3305e8cb3fad1d072" + integrity sha512-IDUSMXs6LOSJBWE++L0lzIbSqHl9KDCfff2x/JSEIDtEUavUnyMYC2ZGay/04Zq4UT8lvd4xNhU4/YHKibAOlw== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.17.0-next.1" - function-bind "^1.1.1" + es-abstract "^1.18.0-next.1" has "^1.0.3" object.pick@^1.3.0: @@ -5270,13 +5160,13 @@ object.pick@^1.3.0: isobject "^3.0.1" object.values@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.1.tgz#68a99ecde356b7e9295a3c5e0ce31dc8c953de5e" - integrity sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA== + version "1.1.2" + resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.2.tgz#7a2015e06fcb0f546bd652486ce8583a4731c731" + integrity sha512-MYC0jvJopr8EK6dPBiO8Nb9mvjdypOachO5REGk6MXzujbBrAisKo3HmdEI6kZDL6fC31Mwee/5YbtMebixeag== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.17.0-next.1" - function-bind "^1.1.1" + es-abstract "^1.18.0-next.1" has "^1.0.3" once@^1.3.0, once@^1.3.1, once@^1.4.0: @@ -5514,10 +5404,10 @@ pg-pool@^3.2.2: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.2.2.tgz#a560e433443ed4ad946b84d774b3f22452694dff" integrity sha512-ORJoFxAlmmros8igi608iVEbQNNZlp89diFVx6yV5v+ehmpMY9sK6QgpmgoXbmkNaBAx8cOOZh9g80kJv1ooyA== -pg-protocol@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.3.0.tgz#3c8fb7ca34dbbfcc42776ce34ac5f537d6e34770" - integrity sha512-64/bYByMrhWULUaCd+6/72c9PMWhiVFs3EVxl9Ct6a3v/U8+rKgqP2w+kKg/BIGgMJyB+Bk/eNivT32Al+Jghw== +pg-protocol@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.4.0.tgz#43a71a92f6fe3ac559952555aa3335c8cb4908be" + integrity sha512-El+aXWcwG/8wuFICMQjM5ZSAm6OWiJicFdNYo+VY3QP+8vI4SvLIWVe51PppTzMhikUJR+PsyIFKqfdXPz/yxA== pg-types@^2.1.0: version "2.2.0" @@ -5531,15 +5421,15 @@ pg-types@^2.1.0: postgres-interval "^1.1.0" pg@^8.4.2: - version "8.4.2" - resolved "https://registry.yarnpkg.com/pg/-/pg-8.4.2.tgz#2aa58166a23391e91d56a7ea57c6d99931c0642a" - integrity sha512-E9FlUrrc7w3+sbRmL1CSw99vifACzB2TjhMM9J5w9D1LIg+6un0jKkpHS1EQf2CWhKhec2bhrBLVMmUBDbjPRQ== + version "8.5.1" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.5.1.tgz#34dcb15f6db4a29c702bf5031ef2e1e25a06a120" + integrity sha512-9wm3yX9lCfjvA98ybCyw2pADUivyNWT/yIP4ZcDVpMN0og70BUWYEGXPCTAQdGTAqnytfRADb7NERrY1qxhIqw== dependencies: buffer-writer "2.0.0" packet-reader "1.0.0" pg-connection-string "^2.4.0" pg-pool "^3.2.2" - pg-protocol "^1.3.0" + pg-protocol "^1.4.0" pg-types "^2.1.0" pgpass "1.x" @@ -5688,9 +5578,9 @@ prettier-linter-helpers@^1.0.0: fast-diff "^1.1.2" prettier@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.1.2.tgz#3050700dae2e4c8b67c4c3f666cdb8af405e1ce5" - integrity sha512-16c7K+x4qVlJg9rEbXl7HEGmQyZlG4R9AgP+oHKRMsMsuk8s+ATStlf1NpDqyBI1HpVyfjLOeMhH2LvuNvV5Vg== + version "2.2.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5" + integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q== pretty-format@^26.0.0, pretty-format@^26.6.2: version "26.6.2" @@ -6090,7 +5980,7 @@ resolve-url@^0.2.1: resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= -resolve@^1.0.0: +resolve@^1.0.0, resolve@^1.10.0, resolve@^1.10.1, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1: version "1.19.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c" integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg== @@ -6098,14 +5988,6 @@ resolve@^1.0.0: is-core-module "^2.1.0" path-parse "^1.0.6" -resolve@^1.10.0, resolve@^1.10.1, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.3.2: - version "1.18.1" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.18.1.tgz#018fcb2c5b207d2a6424aee361c5a266da8f4130" - integrity sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA== - dependencies: - is-core-module "^2.0.0" - path-parse "^1.0.6" - restore-cursor@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" @@ -6134,13 +6016,6 @@ rfdc@^1.1.4: resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.1.4.tgz#ba72cc1367a0ccd9cf81a870b3b58bd3ad07f8c2" integrity sha512-5C9HXdzK8EAqN7JDif30jqsBzavB7wLpaubisuQIGHWf2gUXSpzy6ArX/+Da8RjFpagWsCn+pIgxTMAmKw9Zug== -rimraf@2.6.3: - version "2.6.3" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" - integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA== - dependencies: - glob "^7.1.3" - rimraf@^2.6.1: version "2.7.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" @@ -6148,7 +6023,7 @@ rimraf@^2.6.1: dependencies: glob "^7.1.3" -rimraf@^3.0.0: +rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== @@ -6254,9 +6129,11 @@ semver@7.0.0: integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== semver@7.x, semver@^7.2.1, semver@^7.3.2: - version "7.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" - integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ== + version "7.3.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.4.tgz#27aaa7d2e4ca76452f98d3add093a72c943edc97" + integrity sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw== + dependencies: + lru-cache "^6.0.0" semver@^6.0.0, semver@^6.1.0, semver@^6.3.0: version "6.3.0" @@ -6312,7 +6189,7 @@ shellwords@^0.1.1: resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b" integrity sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww== -side-channel@^1.0.2: +side-channel@^1.0.2, side-channel@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.3.tgz#cdc46b057550bbab63706210838df5d4c19519c3" integrity sha512-A6+ByhlLkksFoUepsGxfj5x1gTSrs+OydsRptUxeNCabQpCFUvcwIczgOigI8vhY/OJCnPnyE9rGiwgvr9cS1g== @@ -6471,9 +6348,9 @@ spdx-expression-parse@^3.0.0: spdx-license-ids "^3.0.0" spdx-license-ids@^3.0.0: - version "3.0.6" - resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.6.tgz#c80757383c28abf7296744998cbc106ae8b854ce" - integrity sha512-+orQK83kyMva3WyPf59k1+Y525csj5JejicWut55zeTWANuN17qSiSLUXWtzHeNWORSvT7GLDJ/E/XiIWoXBTw== + version "3.0.7" + resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.7.tgz#e9c18a410e5ed7e12442a549fbd8afa767038d65" + integrity sha512-U+MTEOO0AiDzxwFvoa4JVnMV6mZlJKk2sBLt90s7G0Gd0Mlknc7kxEn3nuDPNZRta7O2uy8oLcZLVT+4sqNZHQ== split-string@^3.0.1, split-string@^3.0.2: version "3.1.0" @@ -6576,32 +6453,33 @@ string-width@^4.1.0, string-width@^4.2.0: strip-ansi "^6.0.0" string.prototype.matchall@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.2.tgz#48bb510326fb9fdeb6a33ceaa81a6ea04ef7648e" - integrity sha512-N/jp6O5fMf9os0JU3E72Qhf590RSRZU/ungsL/qJUYVTNv7hTG0P/dbPjxINVN9jpscu3nzYwKESU3P3RY5tOg== + version "4.0.3" + resolved "https://registry.yarnpkg.com/string.prototype.matchall/-/string.prototype.matchall-4.0.3.tgz#24243399bc31b0a49d19e2b74171a15653ec996a" + integrity sha512-OBxYDA2ifZQ2e13cP82dWFMaCV9CGF8GzmN4fljBVw5O5wep0lu4gacm1OL6MjROoUnB8VbkWRThqkV2YFLNxw== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.17.0" + es-abstract "^1.18.0-next.1" has-symbols "^1.0.1" internal-slot "^1.0.2" regexp.prototype.flags "^1.3.0" - side-channel "^1.0.2" + side-channel "^1.0.3" string.prototype.trimend@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.2.tgz#6ddd9a8796bc714b489a3ae22246a208f37bfa46" - integrity sha512-8oAG/hi14Z4nOVP0z6mdiVZ/wqjDtWSLygMigTzAb+7aPEDTleeFf+WrF+alzecxIRkckkJVn+dTlwzJXORATw== + version "1.0.3" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.3.tgz#a22bd53cca5c7cf44d7c9d5c732118873d6cd18b" + integrity sha512-ayH0pB+uf0U28CtjlLvL7NaohvR1amUvVZk+y3DYb0Ey2PUV5zPkkKy9+U1ndVEIXO8hNg18eIv9Jntbii+dKw== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.18.0-next.1" string.prototype.trimstart@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.2.tgz#22d45da81015309cd0cdd79787e8919fc5c613e7" - integrity sha512-7F6CdBTl5zyu30BJFdzSTlSlLPwODC23Od+iLoVH8X6+3fvDPPuBVVj9iaB1GOsSTSIgVfsfm27R2FGrAPznWg== + version "1.0.3" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.3.tgz#9b4cb590e123bb36564401d59824298de50fd5aa" + integrity sha512-oBIBUy5lea5tt0ovtOFiEQaBkoBBkyJhZXzJYrSmDo5IUUqbOPvVezuRs/agBIdZ2p2Eo1FD6bD9USyBLfl3xg== dependencies: + call-bind "^1.0.0" define-properties "^1.1.3" - es-abstract "^1.18.0-next.1" string_decoder@^1.1.1: version "1.3.0" @@ -6895,9 +6773,9 @@ ts-jest@^26.4.4: yargs-parser "20.x" ts-node-dev@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/ts-node-dev/-/ts-node-dev-1.0.0.tgz#24a2270d225c29ce269de2a31f88b1b259fc84cb" - integrity sha512-leA/3TgGtnVU77fGngBwVZztqyDRXirytR7dMtMWZS5b2hGpLl+VDnB0F/gf3A+HEPSzS/KwxgXFP7/LtgX4MQ== + version "1.1.1" + resolved "https://registry.yarnpkg.com/ts-node-dev/-/ts-node-dev-1.1.1.tgz#b7602929395b1616b4aa99a3be0a4e121742283d" + integrity sha512-kAO8LUZgXZSY0+PucMPsQ0Bbdv0x+lgbN7j8gcD4PuTI4uKC6YchekaspmYTBNilkiu+rQYkWJA7cK+Q8/B0tQ== dependencies: chokidar "^3.4.0" dateformat "~1.0.4-1.2.3" @@ -6912,11 +6790,12 @@ ts-node-dev@^1.0.0: tsconfig "^7.0.0" ts-node@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-9.0.0.tgz#e7699d2a110cc8c0d3b831715e417688683460b3" - integrity sha512-/TqB4SnererCDR/vb4S/QvSZvzQMJN8daAslg7MeaiHvD8rDZsSfXmNeNumyZZzMned72Xoq/isQljYSt8Ynfg== + version "9.1.1" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-9.1.1.tgz#51a9a450a3e959401bda5f004a72d54b936d376d" + integrity sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg== dependencies: arg "^4.1.0" + create-require "^1.1.0" diff "^4.0.1" make-error "^1.1.1" source-map-support "^0.5.17" @@ -7018,9 +6897,9 @@ typedarray-to-buffer@^3.1.5: is-typedarray "^1.0.0" typescript@^4.0.5: - version "4.0.5" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.5.tgz#ae9dddfd1069f1cb5beb3ef3b2170dd7c1332389" - integrity sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ== + version "4.1.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.1.2.tgz#6369ef22516fe5e10304aae5a5c4862db55380e9" + integrity sha512-thGloWsGH3SOxv1SoY7QojKi0tc+8FnOmiarEGMbd/lar7QOEd3hvlx3Fp5y6FlDUGl9L+pd4n2e+oToGMmhRQ== unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" @@ -7104,9 +6983,9 @@ uuid@^3.3.2: integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== uuid@^8.3.0, uuid@^8.3.1: - version "8.3.1" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.1.tgz#2ba2e6ca000da60fce5a196954ab241131e05a31" - integrity sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg== + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== v8-compile-cache@^2.0.3: version "2.2.0" @@ -7258,17 +7137,10 @@ write-file-atomic@^3.0.0: signal-exit "^3.0.2" typedarray-to-buffer "^3.1.5" -write@1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/write/-/write-1.0.3.tgz#0800e14523b923a387e415123c865616aae0f5c3" - integrity sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig== - dependencies: - mkdirp "^0.5.1" - ws@^7.2.3: - version "7.4.0" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.0.tgz#a5dd76a24197940d4a8bb9e0e152bb4503764da7" - integrity sha512-kyFwXuV/5ymf+IXhS6f0+eAFvydbaBW3zjpT6hUdAh/hbVjTIB5EHBGi0bPoCLSK2wcuz3BrEkB9LrYv1Nm4NQ== + version "7.4.1" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.1.tgz#a333be02696bd0e54cea0434e21dcc8a9ac294bb" + integrity sha512-pTsP8UAfhy3sk1lSk/O/s4tjD0CRwvMnzvwr4OKGX7ZvqZtUyx4KIJB5JWbkykPoc55tixMGgTNoh3k4FkNGFQ== xml-name-validator@^3.0.0: version "3.0.0" @@ -7286,21 +7158,26 @@ xtend@^4.0.0: integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== y18n@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" - integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== + version "4.0.1" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.1.tgz#8db2b83c31c5d75099bb890b23f3094891e247d4" + integrity sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ== -y18n@^5.0.2: +y18n@^5.0.5: version "5.0.5" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.5.tgz#8769ec08d03b1ea2df2500acef561743bbb9ab18" integrity sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg== +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yaml@^1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.0.tgz#3b593add944876077d4d683fee01081bd9fff31e" integrity sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg== -yargs-parser@20.x: +yargs-parser@20.x, yargs-parser@^20.2.2: version "20.2.4" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54" integrity sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA== @@ -7313,11 +7190,6 @@ yargs-parser@^18.1.2: camelcase "^5.0.0" decamelize "^1.2.0" -yargs-parser@^20.2.2: - version "20.2.3" - resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.3.tgz#92419ba867b858c868acf8bae9bf74af0dd0ce26" - integrity sha512-emOFRT9WVHw03QSvN5qor9QQT9+sw5vwxfYweivSMHTcAXPefwVae2FjO7JJjj8hCE4CzPOPeFM83VwT29HCww== - yargs@^15.4.1: version "15.4.1" resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.4.1.tgz#0d87a16de01aee9d8bec2bfbf74f67851730f4f8" @@ -7336,16 +7208,16 @@ yargs@^15.4.1: yargs-parser "^18.1.2" yargs@^16.1.0: - version "16.1.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.1.0.tgz#fc333fe4791660eace5a894b39d42f851cd48f2a" - integrity sha512-upWFJOmDdHN0syLuESuvXDmrRcWd1QafJolHskzaw79uZa7/x53gxQKiR07W59GWY1tFhhU/Th9DrtSfpS782g== + version "16.2.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" + integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== dependencies: cliui "^7.0.2" escalade "^3.1.1" get-caller-file "^2.0.5" require-directory "^2.1.1" string-width "^4.2.0" - y18n "^5.0.2" + y18n "^5.0.5" yargs-parser "^20.2.2" yn@3.1.1: From 0b774edd05576a8b8e953a6bd6e54aa23bbcff3d Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 11 Dec 2020 12:22:49 +0100 Subject: [PATCH 064/218] Fix merge --- src/config.ts | 6 ------ src/server.ts | 1 - src/types.ts | 3 --- 3 files changed, 10 deletions(-) diff --git a/src/config.ts b/src/config.ts index 6777408d..a6b10a2c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -19,9 +19,6 @@ export function getDefaultConfig(): PluginsServerConfig { WORKER_CONCURRENCY: 0, // use all cores TASKS_PER_WORKER: 10, LOG_LEVEL: LogLevel.Info, - STATSD_HOST: null, - STATSD_PORT: 8125, - STATSD_PREFIX: '', SENTRY_DSN: null, STATSD_HOST: null, STATSD_PORT: 8125, @@ -43,9 +40,6 @@ export function getConfigHelp(): Record { WORKER_CONCURRENCY: 'number of concurrent worker threads', TASKS_PER_WORKER: 'number of parallel tasks per worker thread', LOG_LEVEL: 'minimum log level', - STATSD_HOST: 'StatsD host - integration disabled if not provided', - STATSD_PORT: 'StatsD port', - STATSD_PREFIX: 'StatsD prefix', KAFKA_HOSTS: 'comma-delimited Kafka hosts', EE_ENABLED: 'whether Enterprise Edition backend (Kafka + ClickHouse) is enabled', SENTRY_DSN: 'sentry ingestion url', diff --git a/src/server.ts b/src/server.ts index bc18aa41..a013695c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -12,7 +12,6 @@ import Piscina from 'piscina' import { StatsD } from 'hot-shots' import * as Sentry from '@sentry/node' import { delay } from './utils' -import { StatsD } from 'hot-shots' export async function createServer( config: Partial = {}, diff --git a/src/types.ts b/src/types.ts index 5285a577..fa58cf45 100644 --- a/src/types.ts +++ b/src/types.ts @@ -28,9 +28,6 @@ export interface PluginsServerConfig extends Record { DISABLE_WEB: boolean WEB_PORT: number WEB_HOSTNAME: string - STATSD_HOST: string | null - STATSD_PORT: number - STATSD_PREFIX: string LOG_LEVEL: LogLevel SENTRY_DSN: string | null STATSD_HOST: string | null From 6bec3aeb06507cbaa059e8d42f19be4e1102f538 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 14 Dec 2020 07:12:46 +0100 Subject: [PATCH 065/218] Clear Kafka consumption interval on graceful exit --- src/worker/queue.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index f853dfc3..d0a48341 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -61,7 +61,7 @@ function startQueueKafka( eventsProcessor.kafkaConsumer.on('ready', () => { eventsProcessor.kafkaConsumer.subscribe([KAFKA_EVENTS_WAL]) // consume event messages in batches of 1000 every 50 ms - setInterval(() => { + const consumptionInterval = setInterval(() => { eventsProcessor.kafkaConsumer.consume( 1000, async (error: LibrdKafkaError, messages: Message[]): Promise => { @@ -99,6 +99,9 @@ function startQueueKafka( } ) }, 50) + for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { + process.on(signal, () => clearInterval(consumptionInterval)) + } console.info(`โœ… Kafka consumer ready and subscribed to topic ${KAFKA_EVENTS_WAL}!`) }) From 511c40affb61cd375c6daff3553ccc9eb100a1ad Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 14 Dec 2020 07:13:45 +0100 Subject: [PATCH 066/218] Improve logging and error handling --- src/ingestion/process-event.ts | 2 ++ src/types.ts | 20 +++++++++++++++++ src/worker/queue.ts | 39 ++++++++++++++++++++++++++++------ 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index e5925c1d..6fae9b50 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -68,6 +68,7 @@ export class EventsProcessor implements Queue { } stop(): void { + console.info(`โณ Stopping event processing...`) this.kafkaConsumer.disconnect() this.kafkaProducerStreamEvent.destroy() this.kafkaProducerStreamSessionRecording.destroy() @@ -363,6 +364,7 @@ export class EventsProcessor implements Queue { attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith('attr__'))), })) } + const { rows: [team], }: { diff --git a/src/types.ts b/src/types.ts index fa58cf45..93ea6b83 100644 --- a/src/types.ts +++ b/src/types.ts @@ -141,6 +141,26 @@ export interface PropertyUsage { volume: number | null } +// received via Kafka +interface EventMessage { + distinct_id: string + ip: string + site_url: string + team_id: number +} + +export interface RawEventMessage extends EventMessage { + data: string + now: string + sent_at: string // may be an empty string +} + +export interface ParsedEventMessage extends EventMessage { + data: EventData + now: DateTime + sent_at: DateTime | null +} + export interface EventData extends PluginEvent { properties?: Properties timestamp?: string diff --git a/src/worker/queue.ts b/src/worker/queue.ts index d0a48341..8488df07 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -4,7 +4,7 @@ import { LibrdKafkaError, Message } from '@posthog/node-rdkafka' import { DateTime } from 'luxon' import Worker from '../celery/worker' import Client from '../celery/client' -import { EventData, PluginsServer, Queue } from '../types' +import { ParsedEventMessage, PluginsServer, Queue, RawEventMessage } from '../types' import { EventsProcessor } from '../ingestion/process-event' import { KAFKA_EVENTS_WAL } from '../ingestion/topics' @@ -65,20 +65,40 @@ function startQueueKafka( eventsProcessor.kafkaConsumer.consume( 1000, async (error: LibrdKafkaError, messages: Message[]): Promise => { + if (error) { + console.error('โš ๏ธ Error while consuming!') + console.error(error) + Sentry.captureException(error) + } if (messages?.length) { console.info( - `${messages.length} ${messages.length === 1 ? 'message' : 'messages'} consumed from Kafka` + `๐Ÿ• ${Date.now()} ${messages.length} ${ + messages.length === 1 ? 'message' : 'messages' + } consumed from Kafka` ) - } - if (error) { - Sentry.captureException(error) + } else { + return } try { for (const message of messages) { const timer = new Date() - const event = JSON.parse(message.value!.toString()) as EventData - const processedEvent = await processEvent(event) + const rawEventMessage = JSON.parse(message.value!.toString()) as RawEventMessage + const parsedEventMessage: ParsedEventMessage = { + ...rawEventMessage, + data: JSON.parse(rawEventMessage.data), + now: DateTime.fromISO(rawEventMessage.now), + sent_at: rawEventMessage.sent_at ? DateTime.fromISO(rawEventMessage.sent_at) : null, + } + console.info(`Processing event ${parsedEventMessage.data.event} from WAL`) + const processedEvent = await processEvent({ + ...parsedEventMessage, + event: parsedEventMessage.data.event, + properties: parsedEventMessage.data.properties, + now: rawEventMessage.now, + sent_at: rawEventMessage.sent_at, + }) if (processedEvent) { + console.info(`Ingesting event ${parsedEventMessage.data.event}`) const { distinct_id, ip, site_url, team_id, now, sent_at } = processedEvent await eventsProcessor.process_event_ee( distinct_id, @@ -89,11 +109,16 @@ function startQueueKafka( DateTime.fromISO(now), sent_at ? DateTime.fromISO(sent_at) : null ) + console.info(`Ingested event ${parsedEventMessage.data.event}!`) + } else { + console.info(`Discarding event ${parsedEventMessage.data.event}`) } server.statsd?.timing(`${server.STATSD_PREFIX}_posthog_cloud`, timer) } eventsProcessor.kafkaConsumer.commit() } catch (error) { + console.error('โš ๏ธ Error while processing batch of event messages!') + console.error(error) Sentry.captureException(error) } } From a83e254a319e06c0f70d804041723a2385f8ea94 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 14 Dec 2020 07:14:07 +0100 Subject: [PATCH 067/218] Smooth out ingestion errors --- src/ingestion/process-event.ts | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 6fae9b50..6b9e28c8 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -162,7 +162,7 @@ export class EventsProcessor implements Queue { let personFound: Person | undefined personFound = ( await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', [team_id, distinct_id] ) ).rows[0] @@ -174,7 +174,7 @@ export class EventsProcessor implements Queue { } catch { personFound = ( await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', [team_id, distinct_id] ) ).rows[0] @@ -189,7 +189,7 @@ export class EventsProcessor implements Queue { let personFound: Person | undefined personFound = ( await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', [team_id, distinct_id] ) ).rows[0] @@ -208,7 +208,7 @@ export class EventsProcessor implements Queue { } catch { personFound = ( await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', [team_id, distinct_id] ) ).rows[0] @@ -230,14 +230,14 @@ export class EventsProcessor implements Queue { ): Promise { const old_person: Person | undefined = ( await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', [team_id, previous_distinct_id] ) ).rows[0] const new_person: Person | undefined = ( await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE team_id = $1 AND persondistinctid__team_id = $1 AND persondistinctid__distinct_id = $2', + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', [team_id, distinct_id] ) ).rows[0] @@ -416,18 +416,22 @@ export class EventsProcessor implements Queue { team.ingested_event = true save = true } - if (!(event in team.event_names)) { + if (team.event_names && !(event in team.event_names)) { save = true team.event_names.push(event) team.event_names_with_usage.push({ event: event, usage_count: null, volume: null }) } for (const [key, value] of Object.entries(properties)) { - if (!(key in team.event_properties)) { + if (team.event_properties && !(key in team.event_properties)) { team.event_properties.push(key) team.event_properties_with_usage.push({ key: key, usage_count: null, volume: null }) save = true } - if (typeof value === 'number' && !(key in team.event_properties_numerical)) { + if ( + typeof value === 'number' && + team.event_properties_numerical && + !(key in team.event_properties_numerical) + ) { team.event_properties_numerical.push(key) save = true } @@ -437,11 +441,11 @@ export class EventsProcessor implements Queue { 'UPDATE posthog_team SET ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, event_properties_with_usage = $5, event_properties_numerical = $6 WHERE id = $7', [ team.ingested_event, - team.event_names, - team.event_names_with_usage, - team.event_properties, - team.event_names_with_usage, - team.event_properties_numerical, + JSON.stringify(team.event_names), + JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties), + JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties_numerical), team.id, ] ) From cbec4640bfd46951a05786685fe6a273e6fafe89 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 14 Dec 2020 10:31:07 +0100 Subject: [PATCH 068/218] Fix StatsD prefixing --- src/worker/queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 8488df07..7fbeeb10 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -113,7 +113,7 @@ function startQueueKafka( } else { console.info(`Discarding event ${parsedEventMessage.data.event}`) } - server.statsd?.timing(`${server.STATSD_PREFIX}_posthog_cloud`, timer) + server.statsd?.timing(`posthog_cloud`, timer) } eventsProcessor.kafkaConsumer.commit() } catch (error) { From 3a840e4e503823645e3652d71c0a1c3f6738d20c Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 7 Jan 2021 02:34:30 +0100 Subject: [PATCH 069/218] Move @types/luxon to devDependencies --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0700e2ba..792a5821 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "@posthog/node-rdkafka": "2.10.0-6.831-worker-context", "@sentry/node": "^5.29.0", "@sentry/tracing": "^5.29.0", - "@types/luxon": "^1.25.0", "adm-zip": "^0.4.16", "fastify": "^3.8.0", "hot-shots": "^8.2.1", @@ -60,6 +59,7 @@ "@types/adm-zip": "^0.4.33", "@types/ioredis": "^4.17.7", "@types/jest": "^26.0.15", + "@types/luxon": "^1.25.0", "@types/node": "^14.14.6", "@types/node-fetch": "^2.5.7", "@types/node-schedule": "^1.3.1", From e0bdb1c6ac2f07e3747579d07498666467127399 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 8 Jan 2021 12:05:25 +0100 Subject: [PATCH 070/218] Use Kafka.js Producer instead of node-rdkafka --- package.json | 1 - src/ingestion/process-event.ts | 451 +++++++++++++++++++++++++++++++++ src/server.ts | 9 +- src/types.ts | 3 +- yarn.lock | 324 +++++++++++++++++++++-- 5 files changed, 762 insertions(+), 26 deletions(-) diff --git a/package.json b/package.json index 792a5821..51b34426 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ "license": "MIT", "dependencies": { "@google-cloud/bigquery": "^5.5.0", - "@posthog/node-rdkafka": "2.10.0-6.831-worker-context", "@sentry/node": "^5.29.0", "@sentry/tracing": "^5.29.0", "adm-zip": "^0.4.16", diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 4e062e14..ffa4afea 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -4,12 +4,19 @@ import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDist import { castTimestampOrNow, UUID, UUIDT } from '../utils' import { elements_to_string } from './element' import { Event as EventProto } from '../idl/protos' +import { Pool } from 'pg' +import { Producer } from 'kafkajs' +import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' export class EventsProcessor { pluginsServer: PluginsServer + db: Pool + kafkaProducer: Producer constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer + this.db = pluginsServer.db + this.kafkaProducer = pluginsServer.kafkaProducer! } async process_event_ee( @@ -28,5 +35,449 @@ export class EventsProcessor { const person_uuid = new UUIDT() const event_uuid = new UUIDT() + const ts = this.handle_timestamp(data, now, sent_at) + this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) + console.log(`Processing ${data.event}`) + if (data['event'] === '$snapshot') { + await this.create_session_recording_event( + event_uuid, + team_id, + distinct_id, + properties['$session_id'], + ts, + properties['$snapshot_data'] + ) + } else { + await this._capture_ee( + event_uuid, + person_uuid, + ip, + site_url, + team_id, + data['event'], + distinct_id, + properties, + ts, + sent_at + ) + } + } + + handle_timestamp(data: EventData, now: DateTime, sent_at: DateTime | null): DateTime { + if (data['timestamp']) { + if (sent_at) { + // sent_at - timestamp == now - x + // x = now + (timestamp - sent_at) + try { + // timestamp and sent_at must both be in the same format: either both with or both without timezones + // otherwise we can't get a diff to add to now + return now.plus(DateTime.fromISO(data['timestamp']).diff(sent_at)) + } catch (error) { + console.error(error) + } + } + return DateTime.fromISO(data['timestamp']) + } + if (data['offset']) { + return now.minus(Duration.fromMillis(data['offset'])) + } + return now + } + + async handle_identify_or_alias( + event: string, + properties: Properties, + distinct_id: string, + team_id: number + ): Promise { + if (event === '$create_alias') { + await this._alias(properties['alias'], distinct_id, team_id) + } else if (event === '$identify') { + if (properties['$anon_distinct_id']) { + await this._alias(properties['$anon_distinct_id'], distinct_id, team_id) + } + if (properties['$set']) { + this._update_person_properties(team_id, distinct_id, properties['$set']) + } + this._set_is_identified(team_id, distinct_id) + } + } + + async _set_is_identified(team_id: number, distinct_id: string, is_identified = true): Promise { + let personFound: Person | undefined + personFound = ( + await this.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + if (!personFound) { + try { + const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, true, new UUIDT()) + this.add_distinct_id(personCreated, distinct_id) + // Catch race condition where in between getting and creating, another request already created this person + } catch { + personFound = ( + await this.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + } + } + if (personFound && !personFound.is_identified) { + await this.db.query('UPDATE posthog_person SET is_identified = 1 WHERE id = $1', [personFound.id]) + } + } + + async _update_person_properties(team_id: number, distinct_id: string, properties: Properties): Promise { + let personFound: Person | undefined + personFound = ( + await this.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + if (!personFound) { + try { + const personCreated = await this.create_person( + DateTime.utc(), + properties, + team_id, + null, + false, + new UUIDT() + ) + await this.add_distinct_id(personCreated, distinct_id) + // Catch race condition where in between getting and creating, another request already created this person + } catch { + personFound = ( + await this.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + } + } + if (personFound) { + this.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ + { ...personFound.properties, ...properties }, + personFound.id, + ]) + } + } + + async _alias( + previous_distinct_id: string, + distinct_id: string, + team_id: number, + retry_if_failed = true + ): Promise { + const old_person: Person | undefined = ( + await this.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', + [team_id, previous_distinct_id] + ) + ).rows[0] + + const new_person: Person | undefined = ( + await this.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', + [team_id, distinct_id] + ) + ).rows[0] + + if (old_person && !new_person) { + try { + this.add_distinct_id(old_person, distinct_id) + // Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + } catch { + // integrity error + if (retry_if_failed) { + // run everything again to merge the users if needed + this._alias(previous_distinct_id, distinct_id, team_id, false) + } + } + return + } + + if (!old_person && new_person) { + try { + this.add_distinct_id(new_person, previous_distinct_id) + // Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + } catch { + // integrity error + if (retry_if_failed) { + // run everything again to merge the users if needed + this._alias(previous_distinct_id, distinct_id, team_id, false) + } + } + return + } + + if (!old_person && !new_person) { + try { + const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, false, new UUIDT()) + this.add_distinct_id(personCreated, distinct_id) + this.add_distinct_id(personCreated, previous_distinct_id) + // Catch race condition where in between getting and creating, another request already created this user. + } catch { + // integrity error + if (retry_if_failed) { + // try once more, probably one of the two persons exists now + this._alias(previous_distinct_id, distinct_id, team_id, false) + } + } + return + } + + if (old_person && new_person && old_person.id !== new_person.id) { + this.merge_people(new_person, [old_person]) + } + } + + async merge_people(merge_into: Person, people_to_merge: Person[]): Promise { + let first_seen = merge_into.created_at + + // merge the properties + for (const other_person of people_to_merge) { + merge_into.properties = { ...other_person.properties, ...merge_into.properties } + if (other_person.created_at < first_seen) { + // Keep the oldest created_at (i.e. the first time we've seen this person) + first_seen = other_person.created_at + } + } + + await this.db.query('UPDATE posthog_person SET created_at = $1 WHERE id = $2', [ + first_seen.toISO(), + merge_into.id, + ]) + + // merge the distinct_ids + for (const other_person of people_to_merge) { + const other_person_distinct_ids: PersonDistinctId[] = ( + await this.db.query('SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', [ + other_person, + merge_into.team_id, + ]) + ).rows + for (const person_distinct_id of other_person_distinct_ids) { + await this.db.query('UPDATE posthog_persondistinctid SET person_id = $1 WHERE id = $2', [ + merge_into.id, + person_distinct_id.id, + ]) + } + + const other_person_cohort_ids: CohortPeople[] = ( + await this.db.query('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [other_person.id]) + ).rows + for (const person_cohort_id of other_person_cohort_ids) { + await this.db.query('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ + merge_into.id, + person_cohort_id.id, + ]) + } + + await this.db.query('DELETE FROM posthog_person WHERE id = $1', [other_person.id]) + } + } + + async _capture_ee( + event_uuid: UUID, + person_uuid: UUID, + ip: string, + site_url: string, + team_id: number, + event: string, + distinct_id: string, + properties: Properties, + timestamp: DateTime, + sent_at: DateTime | null + ): Promise { + const elements: Record[] | undefined = properties['$elements'] + let elements_list: Element[] = [] + if (elements && elements.length) { + delete properties['$elements'] + elements_list = elements.map((el) => ({ + text: el['$el_text'] ? el['$el_text'].slice(0, 400) : null, + tag_name: el['tag_name'], + href: el['attr__href'] ? el['attr__href'].slice(0, 2048) : null, + attr_class: el['attr__class'] ? el['attr__class'].split(' ') : null, + attr_id: el['attr__id'], + nth_child: el['nth_child'], + nth_of_type: el['nth_of_type'], + attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith('attr__'))), + })) + } + + const { + rows: [team], + }: { + rows: Team[] + } = await this.db.query( + 'SELECT slack_incoming_webhook, event_names, event_properties, event_names_with_usage, event_properties_with_usage, anonymize_ips FROM posthog_team WHERE id = $1', + [team_id] + ) + + if (!team.anonymize_ips && !('$ip' in properties)) { + properties['$ip'] = ip + } + + this.store_names_and_properties(team, event, properties) + + const { + rows: [{ pdiCount }], + }: { + rows: { pdiCount: number }[] + } = await this.db.query( + 'SELECT COUNT(*) AS pdiCount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', + [team_id, distinct_id] + ) + + if (!pdiCount) { + // Catch race condition where in between getting and creating, another request already created this user + try { + const personCreated: Person = await this.create_person( + sent_at || DateTime.utc(), + {}, + team_id, + null, + false, + person_uuid.toString() + ) + await this.add_distinct_id(personCreated, distinct_id) + } catch {} + } + + await this.create_event(event_uuid, event, team, distinct_id, properties, timestamp, elements_list) + } + + async store_names_and_properties(team: Team, event: string, properties: Properties): Promise { + // In _capture we only prefetch a couple of fields in Team to avoid fetching too much data + let save = false + if (!team.ingested_event) { + // First event for the team captured + // TODO: capture "first team event ingested" + team.ingested_event = true + save = true + } + if (team.event_names && !(event in team.event_names)) { + save = true + team.event_names.push(event) + team.event_names_with_usage.push({ event: event, usage_count: null, volume: null }) + } + for (const [key, value] of Object.entries(properties)) { + if (team.event_properties && !(key in team.event_properties)) { + team.event_properties.push(key) + team.event_properties_with_usage.push({ key: key, usage_count: null, volume: null }) + save = true + } + if ( + typeof value === 'number' && + team.event_properties_numerical && + !(key in team.event_properties_numerical) + ) { + team.event_properties_numerical.push(key) + save = true + } + } + if (save) { + await this.db.query( + 'UPDATE posthog_team SET ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, event_properties_with_usage = $5, event_properties_numerical = $6 WHERE id = $7', + [ + team.ingested_event, + JSON.stringify(team.event_names), + JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties), + JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties_numerical), + team.id, + ] + ) + } + } + + async create_person( + created_at: DateTime, + properties: Properties, + team_id: number, + is_user_id: number | null, + is_identified: boolean, + uuid: UUID | string + ): Promise { + return ( + await this.db.query( + 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6)', + [created_at.toISO(), properties, team_id, is_user_id, is_identified, uuid.toString()] + ) + ).rows[0] + } + + async add_distinct_id(person: Person, distinct_id: string): Promise { + await this.db.query( + 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3)', + [distinct_id, person.id, person.team_id] + ) + } + + async create_event( + event_uuid: UUID, + event: string, + team: Team, + distinctId: string, + properties?: Properties, + timestamp?: DateTime | string, + elements?: Element[] + ): Promise { + const timestampString = castTimestampOrNow(timestamp) + const elementsChain = elements && elements.length ? elements_to_string(elements) : '' + const eventUuidString = event_uuid.toString() + + const message = EventProto.create({ + uuid: eventUuidString, + event, + properties: JSON.stringify(properties ?? {}), + timestamp: timestampString, + teamId: team.id, + distinctId, + elementsChain, + createdAt: timestampString, + }) + console.log(`Producing ${event}`) + await this.kafkaProducer.send({ + topic: KAFKA_EVENTS, + messages: [{ key: eventUuidString, value: Buffer.from(EventProto.encode(message).finish()) }], + }) + + return eventUuidString + } + + async create_session_recording_event( + uuid: UUID, + team_id: number, + distinct_id: string, + session_id: string, + timestamp: DateTime | string, + snapshot_data: Record + ): Promise { + const timestampString = castTimestampOrNow(timestamp) + const uuidString = uuid.toString() + + const data = { + uuid: uuidString, + team_id: team_id, + distinct_id: distinct_id, + session_id: session_id, + snapshot_data: JSON.stringify(snapshot_data), + timestamp: timestampString, + created_at: timestampString, + } + + await this.kafkaProducer.send({ + topic: KAFKA_SESSION_RECORDING_EVENTS, + messages: [{ key: uuidString, value: Buffer.from(JSON.stringify(data)) }], + }) + + return uuidString } } diff --git a/src/server.ts b/src/server.ts index facbffd7..08e0561b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,7 +1,7 @@ import { Pool } from 'pg' import * as schedule from 'node-schedule' import Redis from 'ioredis' -import { Kafka, logLevel } from 'kafkajs' +import { Kafka, logLevel, Producer } from 'kafkajs' import { FastifyInstance } from 'fastify' import { PluginsServer, PluginsServerConfig, Queue } from './types' import { startQueue } from './worker/queue' @@ -45,6 +45,7 @@ export async function createServer( }) let kafka: Kafka | undefined + let kafkaProducer: Producer | undefined if (serverConfig.KAFKA_ENABLED) { if (!serverConfig.KAFKA_HOSTS) { throw new Error('You must set KAFKA_HOSTS to process events from Kafka!') @@ -64,6 +65,7 @@ export async function createServer( } : undefined, }) + kafkaProducer = kafka.producer() } let statsd: StatsD | undefined @@ -87,6 +89,7 @@ export async function createServer( db, redis, kafka, + kafkaProducer, statsd, plugins: new Map(), pluginConfigs: new Map(), @@ -128,6 +131,7 @@ export async function startPluginsServer( let pingJob: schedule.Job | undefined let statsJob: schedule.Job | undefined let piscina: Piscina | undefined + let kafkaProducer: Producer | undefined let queue: Queue | undefined let closeServer: () => Promise | undefined let stopSchedule: () => Promise | undefined @@ -150,6 +154,7 @@ export async function startPluginsServer( await stopFastifyInstance(fastifyInstance!) } await queue?.stop() + await kafkaProducer?.disconnect() pubSub?.disconnect() pingJob && schedule.cancelJob(pingJob) statsJob && schedule.cancelJob(statsJob) @@ -172,6 +177,8 @@ export async function startPluginsServer( } ;[server, closeServer] = await createServer(serverConfig, null) + await server.kafkaProducer?.connect() + piscina = makePiscina(serverConfig) const processEvent = (event: PluginEvent) => { if ((piscina?.queueSize || 0) > (server?.WORKER_CONCURRENCY || 4) * (server?.WORKER_CONCURRENCY || 4)) { diff --git a/src/types.ts b/src/types.ts index 45baf7e4..1335cc93 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,6 @@ import { Pool } from 'pg' import { Redis } from 'ioredis' -import { Kafka } from 'kafkajs' +import { Kafka, Producer } from 'kafkajs' import { PluginEvent, PluginAttachment, PluginConfigSchema } from '@posthog/plugin-scaffold' import { VM } from 'vm2' import { DateTime } from 'luxon' @@ -53,6 +53,7 @@ export interface PluginsServer extends PluginsServerConfig { db: Pool redis: Redis kafka: Kafka | undefined + kafkaProducer: Producer | undefined statsd: StatsD | undefined // currently enabled plugin status plugins: Map diff --git a/yarn.lock b/yarn.lock index ed27cf0a..51fee7f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -904,6 +904,56 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" +"@google-cloud/bigquery@^5.5.0": + version "5.5.0" + resolved "https://registry.yarnpkg.com/@google-cloud/bigquery/-/bigquery-5.5.0.tgz#cfe72f8d056abd9c8289c724150d1082a0a6367d" + integrity sha512-hMje0+Fd1B/b29yMUGvnTUXEaUQQD1zyvYjGEu/HCh9w2ek47DQr1zVrHdstlpstUZfaeRCjxK/c3HY6wWrVFg== + dependencies: + "@google-cloud/common" "^3.1.0" + "@google-cloud/paginator" "^3.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.1" + big.js "^6.0.0" + duplexify "^4.0.0" + extend "^3.0.2" + is "^3.3.0" + p-event "^4.1.0" + stream-events "^1.0.5" + uuid "^8.0.0" + +"@google-cloud/common@^3.1.0": + version "3.5.0" + resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.5.0.tgz#0959e769e8075a06eb0823cc567eef00fd0c2d02" + integrity sha512-10d7ZAvKhq47L271AqvHEd8KzJqGU45TY+rwM2Z3JHuB070FeTi7oJJd7elfrnKaEvaktw3hH2wKnRWxk/3oWQ== + dependencies: + "@google-cloud/projectify" "^2.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.1" + duplexify "^4.1.1" + ent "^2.2.0" + extend "^3.0.2" + google-auth-library "^6.1.1" + retry-request "^4.1.1" + teeny-request "^7.0.0" + +"@google-cloud/paginator@^3.0.0": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@google-cloud/paginator/-/paginator-3.0.5.tgz#9d6b96c421a89bd560c1bc2c197c7611ef21db6c" + integrity sha512-N4Uk4BT1YuskfRhKXBs0n9Lg2YTROZc6IMpkO/8DIHODtm5s3xY8K5vVBo23v/2XulY3azwITQlYWgT4GdLsUw== + dependencies: + arrify "^2.0.0" + extend "^3.0.2" + +"@google-cloud/projectify@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@google-cloud/projectify/-/projectify-2.0.1.tgz#13350ee609346435c795bbfe133a08dfeab78d65" + integrity sha512-ZDG38U/Yy6Zr21LaR3BTiiLtpJl6RkPS/JwoRT453G+6Q1DhlV0waNf8Lfu+YVYGIIxgKnLayJRfYlFJfiI8iQ== + +"@google-cloud/promisify@^2.0.0": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.3.tgz#f934b5cdc939e3c7039ff62b9caaf59a9d89e3a8" + integrity sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw== + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" @@ -1129,13 +1179,10 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" -"@posthog/node-rdkafka@2.10.0-6.831-worker-context": - version "2.10.0-6.831-worker-context" - resolved "https://registry.yarnpkg.com/@posthog/node-rdkafka/-/node-rdkafka-2.10.0-6.831-worker-context.tgz#8b361f25f5c7633208331b31bf54583ab4848f61" - integrity sha512-FWzVTFrAeerw9r8QPw+FOjZgyq0GO0Z3z0YzZGHzTWSEZMhpiLRGLiAn73U2OxiKHNFTWqO5Afu6vKoIPwj1kQ== - dependencies: - bindings "^1.3.1" - nan "^2.14.0" +"@posthog/plugin-scaffold@0.2.5": + version "0.2.5" + resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.5.tgz#9fd33b02b1dd8877d0c726cb5087f25556977bfc" + integrity sha512-mWwVXQL86xrTFz5vegSikiJlz3cmQPJzP29Pv0qtXzm+F4AvzDHhMPbHjFnQQpoqvglKLe768OVGuYaDiinHDA== "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" @@ -1272,6 +1319,11 @@ dependencies: "@sinonjs/commons" "^1.7.0" +"@tootallnate/once@1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" + integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== + "@types/adm-zip@^0.4.33": version "0.4.33" resolved "https://registry.yarnpkg.com/@types/adm-zip/-/adm-zip-0.4.33.tgz#ea5b94f771443f655613b64f920c0555867200dd" @@ -1312,6 +1364,11 @@ dependencies: "@babel/types" "^7.3.0" +"@types/bluebird@*": + version "3.5.33" + resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.33.tgz#d79c020f283bd50bd76101d7d300313c107325fc" + integrity sha512-ndEo1xvnYeHxm7I/5sF6tBvnsA4Tdi3zj1keRKRs12SP+2ye2A27NDJ1B6PqkfMbGAcT+mqQVqbZRIrhfOp5PQ== + "@types/cookiejar@*": version "2.1.2" resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.2.tgz#66ad9331f63fe8a3d3d9d8c6e3906dd10f6446e8" @@ -1431,6 +1488,13 @@ resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.1.5.tgz#b6ab3bba29e16b821d84e09ecfaded462b816b00" integrity sha512-UEyp8LwZ4Dg30kVU2Q3amHHyTn1jEdhCIE59ANed76GaT1Vp76DD3ZWSAxgCrw6wJ0TqeoBpqmfUHiUDPs//HQ== +"@types/redlock@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@types/redlock/-/redlock-4.0.1.tgz#cfa620968f2ebe795f3abeef8e9128f071eeef85" + integrity sha512-YrPEPHOgLlWtsg/Ocv/gthGDQpx3HXFPhBvCNuBKBKUoMBzvCjCTC95dmJ0WLZgO+fgTxGQFyk6ZO/+/zG5mRg== + dependencies: + "@types/bluebird" "*" + "@types/stack-utils@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.0.tgz#7036640b4e21cc2f259ae826ce843d277dad8cff" @@ -1560,6 +1624,13 @@ abab@^2.0.3: resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" integrity sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + abstract-logging@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839" @@ -1752,6 +1823,11 @@ array.prototype.flatmap@^1.2.3: es-abstract "^1.18.0-next.1" function-bind "^1.1.1" +arrify@^2.0.0, arrify@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + asn1@~0.2.3: version "0.2.4" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" @@ -1909,7 +1985,7 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= -base64-js@^1.2.0, base64-js@^1.3.1: +base64-js@^1.2.0, base64-js@^1.3.0, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -1934,6 +2010,16 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +big.js@^6.0.0: + version "6.0.3" + resolved "https://registry.yarnpkg.com/big.js/-/big.js-6.0.3.tgz#8b4d99ac7023668e0e465d3f78c23b8ac29ad381" + integrity sha512-n6yn1FyVL1EW2DBAr4jlU/kObhRzmr+NNRESl65VIOT8WBJj/Kezpx2zFdhJUqYI6qrtTW7moCStYL5VxeVdPA== + +bignumber.js@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.0.1.tgz#8d7ba124c882bfd8e43260c67475518d0689e4e5" + integrity sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA== + binary-extensions@^1.0.0: version "1.13.1" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.1.tgz#598afe54755b2868a5330d2aff9d4ebb53209b65" @@ -1944,7 +2030,7 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== -bindings@^1.3.0, bindings@^1.3.1: +bindings@^1.3.0: version "1.5.0" resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== @@ -1960,6 +2046,11 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" +bluebird@^3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" + integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -2021,6 +2112,11 @@ bser@2.1.1: dependencies: node-int64 "^0.4.0" +buffer-equal-constant-time@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" + integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk= + buffer-from@1.x, buffer-from@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" @@ -2560,6 +2656,16 @@ domexception@^2.0.1: dependencies: webidl-conversions "^5.0.0" +duplexify@^4.0.0, duplexify@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.1.tgz#7027dc374f157b122a8ae08c2d3ea4d2d953aa61" + integrity sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA== + dependencies: + end-of-stream "^1.4.1" + inherits "^2.0.3" + readable-stream "^3.1.1" + stream-shift "^1.0.0" + dynamic-dedupe@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz#06e44c223f5e4e94d78ef9db23a6515ce2f962a1" @@ -2575,6 +2681,13 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" +ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: + version "1.0.11" + resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" + integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== + dependencies: + safe-buffer "^5.0.1" + electron-to-chromium@^1.3.621: version "1.3.622" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.622.tgz#9726bd2e67a5462154750ce9701ca6af07d07877" @@ -2609,6 +2722,11 @@ enquirer@^2.3.5, enquirer@^2.3.6: dependencies: ansi-colors "^4.1.1" +ent@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d" + integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0= + error-ex@^1.2.0, error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -2959,6 +3077,11 @@ event-emitter@~0.3.5: d "1" es5-ext "~0.10.14" +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + eventemitter-asyncresource@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b" @@ -3049,7 +3172,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@~3.0.2: +extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== @@ -3134,6 +3257,11 @@ fast-safe-stringify@^2.0.7: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743" integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA== +fast-text-encoding@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" + integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig== + fastify-error@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/fastify-error/-/fastify-error-0.2.0.tgz#9a1c28d4f42b6259e7a549671c8e5e2d85660634" @@ -3367,6 +3495,25 @@ functional-red-black-tree@^1.0.1: resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= +gaxios@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.1.0.tgz#e8ad466db5a4383c70b9d63bfd14dfaa87eb0099" + integrity sha512-vb0to8xzGnA2qcgywAjtshOKKVDf2eQhJoiL6fHhgW5tVN7wNk7egnYIO9zotfn3lQ3De1VPdf7V5/BWfCtCmg== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.3.0" + +gcp-metadata@^4.2.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.2.1.tgz#31849fbcf9025ef34c2297c32a89a1e7e9f2cd62" + integrity sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw== + dependencies: + gaxios "^4.0.0" + json-bigint "^1.0.0" + gensync@^1.0.0-beta.1: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -3483,6 +3630,28 @@ globby@^11.0.1: merge2 "^1.3.0" slash "^3.0.0" +google-auth-library@^6.1.1: + version "6.1.4" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.4.tgz#bc70c4f3b6681ae5273343466bcef37577b7ee44" + integrity sha512-q0kYtGWnDd9XquwiQGAZeI2Jnglk7NDi0cChE4tWp6Kpo/kbqnt9scJb0HP+/xqt03Beqw/xQah1OPrci+pOxw== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + +google-p12-pem@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.0.3.tgz#673ac3a75d3903a87f05878f3c75e06fc151669e" + integrity sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA== + dependencies: + node-forge "^0.10.0" + graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.2.4: version "4.2.4" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" @@ -3493,6 +3662,16 @@ growly@^1.3.0: resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081" integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE= +gtoken@^5.0.4: + version "5.1.0" + resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.1.0.tgz#4ba8d2fc9a8459098f76e7e8fd7beaa39fda9fe4" + integrity sha512-4d8N6Lk8TEAHl9vVoRVMh9BNOKWVgl2DdNtr3428O75r3QFrF/a5MMu851VmK0AA8+iSvbwRv69k5XnMLURGhg== + dependencies: + gaxios "^4.0.0" + google-p12-pem "^3.0.3" + jws "^4.0.0" + mime "^2.2.0" + har-schema@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" @@ -3597,6 +3776,15 @@ html-escaper@^2.0.0: resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== +http-proxy-agent@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" + integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== + dependencies: + "@tootallnate/once" "1" + agent-base "6" + debug "4" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -3999,6 +4187,11 @@ is-wsl@^2.2.0: dependencies: is-docker "^2.0.0" +is@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/is/-/is-3.3.0.tgz#61cff6dd3c4193db94a3d62582072b44e5645d79" + integrity sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg== + isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -4500,6 +4693,13 @@ jsesc@~0.5.0: resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= +json-bigint@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" + integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ== + dependencies: + bignumber.js "^9.0.0" + json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" @@ -4557,6 +4757,28 @@ jsprim@^1.2.2: array-includes "^3.1.1" object.assign "^4.1.1" +jwa@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" + integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + +kafkajs@^1.15.0: + version "1.15.0" + resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-1.15.0.tgz#a5ada0d933edca2149177393562be6fb0875ec3a" + integrity sha512-yjPyEnQCkPxAuQLIJnY5dI+xnmmgXmhuOQ1GVxClG5KTOV/rJcW1qA3UfvyEJKTp/RTSqQnUR3HJsKFvHyTpNg== + kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -4892,6 +5114,11 @@ mime-types@^2.1.12, mime-types@~2.1.19: dependencies: mime-db "1.44.0" +mime@^2.2.0: + version "2.4.7" + resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.7.tgz#962aed9be0ed19c91fd7dc2ece5d7f4e89a90d74" + integrity sha512-dhNd1uA2u397uQk3Nv5LM4lm93WYDUXFn3Fu291FJerns4jyTudqhIWe4W04YLy7Uk1tm1Ore04NpjRvQp/NPA== + mime@^2.4.6: version "2.4.6" resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" @@ -4949,7 +5176,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -nan@^2.13.2, nan@^2.14.0: +nan@^2.13.2: version "2.14.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== @@ -4999,11 +5226,16 @@ node-addon-api@^3.0.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.0.2.tgz#04bc7b83fd845ba785bb6eae25bc857e1ef75681" integrity sha512-+D4s2HCnxPd5PjjI0STKwncjXTUKKqm74MDMz9OPXavjsGmjkvwgLtA5yoxJUdmpj52+2u+RrXgPipahKczMKg== -node-fetch@^2.6.1: +node-fetch@^2.3.0, node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== +node-forge@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" + integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== + node-gyp-build@^4.2.2: version "4.2.3" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739" @@ -5222,6 +5454,13 @@ p-each-series@^2.1.0: resolved "https://registry.yarnpkg.com/p-each-series/-/p-each-series-2.2.0.tgz#105ab0357ce72b202a8a8b94933672657b5e2a9a" integrity sha512-ycIL2+1V32th+8scbpTvyHNaHe02z0sjgh91XXjAk+ZeXoPN4Z46DVUnzdso0aX4KckKw0FNNFHdjZ2UsZvxiA== +p-event@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/p-event/-/p-event-4.2.0.tgz#af4b049c8acd91ae81083ebd1e6f5cae2044c1b5" + integrity sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ== + dependencies: + p-timeout "^3.1.0" + p-finally@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" @@ -5267,6 +5506,13 @@ p-map@^4.0.0: dependencies: aggregate-error "^3.0.0" +p-timeout@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" + p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -5550,16 +5796,6 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" -posthog-js-lite@^0.0.5: - version "0.0.5" - resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.5.tgz#984a619190d1c4ef003cb81194d0a6276491d2e8" - integrity sha512-xHVZ9qbBdoqK7G1JVA4k6nheWu50aiEwjJIPQw7Zhi705aX5wr2W8zxdmvtadeEE5qbJNg+/uZ7renOQoqcbJw== - -posthog-plugins@0.2.4: - version "0.2.4" - resolved "https://registry.yarnpkg.com/posthog-plugins/-/posthog-plugins-0.2.4.tgz#e4050795e7e14ada29de301f6b15e900bbad2ce8" - integrity sha512-M+t4Juu/Vr/GSRwjNDXunarPck7YXEqF6AKb4u5tRrOBI2VIyHjJiLtsIj5SwVsl4VKpMyyfahV/0No7ByT3TA== - prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -5815,6 +6051,13 @@ redis-parser@^3.0.0: dependencies: redis-errors "^1.0.0" +redlock@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/redlock/-/redlock-4.2.0.tgz#c26590768559afd5fff76aa1133c94b411ff4f5f" + integrity sha512-j+oQlG+dOwcetUt2WJWttu4CZVeRzUrcVcISFmEmfyuwCVSJ93rDT7YSgg7H7rnxwoRyk/jU46kycVka5tW7jA== + dependencies: + bluebird "^3.7.2" + regenerate-unicode-properties@^8.2.0: version "8.2.0" resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec" @@ -6006,6 +6249,13 @@ ret@~0.2.0: resolved "https://registry.yarnpkg.com/ret/-/ret-0.2.2.tgz#b6861782a1f4762dce43402a71eb7a283f44573c" integrity sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ== +retry-request@^4.1.1: + version "4.1.3" + resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-4.1.3.tgz#d5f74daf261372cff58d08b0a1979b4d7cab0fde" + integrity sha512-QnRZUpuPNgX0+D1xVxul6DbJ9slvo4Rm6iV/dn63e048MvGbUZiKySVt6Tenp04JqmchxjiLltGerOJys7kJYQ== + dependencies: + debug "^4.1.1" + reusify@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" @@ -6416,6 +6666,18 @@ stealthy-require@^1.1.1: resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks= +stream-events@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/stream-events/-/stream-events-1.0.5.tgz#bbc898ec4df33a4902d892333d47da9bf1c406d5" + integrity sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg== + dependencies: + stubs "^3.0.0" + +stream-shift@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" + integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== + string-argv@0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da" @@ -6562,6 +6824,11 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== +stubs@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/stubs/-/stubs-3.0.0.tgz#e8d2ba1fa9c90570303c030b6900f7d5f89abe5b" + integrity sha1-6NK6H6nJBXAwPAMLaQD31fiavls= + superagent@6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/superagent/-/superagent-6.1.0.tgz#09f08807bc41108ef164cfb4be293cebd480f4a6" @@ -6635,6 +6902,17 @@ tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" +teeny-request@^7.0.0: + version "7.0.1" + resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-7.0.1.tgz#bdd41fdffea5f8fbc0d29392cb47bec4f66b2b4c" + integrity sha512-sasJmQ37klOlplL4Ia/786M5YlOcoLGQyq2TE4WHSRupbAuDaQW0PfVxV4MtdBtRJ4ngzS+1qim8zP6Zp35qCw== + dependencies: + http-proxy-agent "^4.0.0" + https-proxy-agent "^5.0.0" + node-fetch "^2.6.1" + stream-events "^1.0.5" + uuid "^8.0.0" + terminal-link@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/terminal-link/-/terminal-link-2.1.1.tgz#14a64a27ab3c0df933ea546fba55f2d078edc994" @@ -6982,7 +7260,7 @@ uuid@^3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== -uuid@^8.3.0, uuid@^8.3.1: +uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== From e948f9381e84de0dd3494b56d0c703f75c365ccb Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 11 Jan 2021 16:56:41 +0100 Subject: [PATCH 071/218] Use EventProto#encodeDelimited instead of #encode --- src/ingestion/process-event.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index ffa4afea..ab30347c 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -446,7 +446,7 @@ export class EventsProcessor { console.log(`Producing ${event}`) await this.kafkaProducer.send({ topic: KAFKA_EVENTS, - messages: [{ key: eventUuidString, value: Buffer.from(EventProto.encode(message).finish()) }], + messages: [{ value: EventProto.encodeDelimited(message).finish() as Buffer }], }) return eventUuidString @@ -475,7 +475,7 @@ export class EventsProcessor { await this.kafkaProducer.send({ topic: KAFKA_SESSION_RECORDING_EVENTS, - messages: [{ key: uuidString, value: Buffer.from(JSON.stringify(data)) }], + messages: [{ value: Buffer.from(JSON.stringify(data)) }], }) return uuidString From d8e0b1e1fec18b51bbd30f31b75aa31c81e92833 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 16:29:03 +0100 Subject: [PATCH 072/218] Start UUIDT series from 0 instead of 1 --- src/utils.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 4ca9e2b9..09ac63f6 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -230,14 +230,13 @@ export class UUIDT extends UUID { /** Get per-millisecond series integer in range [0-65536). */ static getSeries(unixTimeMs: number): number { - const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) + const series = UUIDT.currentSeriesPerMs.get(unixTimeMs) ?? 0 if (UUIDT.currentSeriesPerMs.size > 10_000) { // Clear class dict periodically UUIDT.currentSeriesPerMs.clear() } - const nextSeries = typeof series === 'number' ? (series + 1) % 65_536 : 0 - UUIDT.currentSeriesPerMs.set(unixTimeMs, nextSeries) - return nextSeries + UUIDT.currentSeriesPerMs.set(unixTimeMs, (series + 1) % 65_536) + return series } constructor(unixTimeMs?: number) { From 77080d568324060fde22a09e29357fbbcf4b8f7f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 16:29:52 +0100 Subject: [PATCH 073/218] Use event UUID from Django server --- src/ingestion/process-event.ts | 9 +++++---- src/worker/queue.ts | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index ab30347c..fc03229e 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -26,7 +26,8 @@ export class EventsProcessor { data: PluginEvent, team_id: number, now: DateTime, - sent_at: DateTime | null + sent_at: DateTime | null, + event_uuid: UUID ): Promise { const properties: Properties = data.properties ?? {} if (data['$set']) { @@ -34,7 +35,7 @@ export class EventsProcessor { } const person_uuid = new UUIDT() - const event_uuid = new UUIDT() + const ts = this.handle_timestamp(data, now, sent_at) this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) console.log(`Processing ${data.event}`) @@ -446,7 +447,7 @@ export class EventsProcessor { console.log(`Producing ${event}`) await this.kafkaProducer.send({ topic: KAFKA_EVENTS, - messages: [{ value: EventProto.encodeDelimited(message).finish() as Buffer }], + messages: [{ key: eventUuidString, value: EventProto.encodeDelimited(message).finish() as Buffer }], }) return eventUuidString @@ -475,7 +476,7 @@ export class EventsProcessor { await this.kafkaProducer.send({ topic: KAFKA_SESSION_RECORDING_EVENTS, - messages: [{ value: Buffer.from(JSON.stringify(data)) }], + messages: [{ key: uuidString, value: Buffer.from(JSON.stringify(data)) }], }) return uuidString diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 4f4ddf88..6f9f3a6e 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -3,8 +3,9 @@ import * as Sentry from '@sentry/node' import { DateTime } from 'luxon' import Worker from '../celery/worker' import Client from '../celery/client' -import { PluginsServer, Queue, RawEventMessage } from '../types' +import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' +import { UUID } from '../utils' export function startQueue( server: PluginsServer, @@ -67,7 +68,7 @@ function startQueueKafka( ): Queue { const kafkaQueue = new KafkaQueue(server, processEventBatch, async (event: PluginEvent) => { const singleIngestionTimer = new Date() - const { distinct_id, ip, site_url, team_id, now, sent_at } = event + const { distinct_id, ip, site_url, team_id, now, sent_at, uuid } = event await server.eventsProcessor.process_event_ee( distinct_id, ip, @@ -75,7 +76,8 @@ function startQueueKafka( event, team_id, DateTime.fromISO(now), - sent_at ? DateTime.fromISO(sent_at) : null + sent_at ? DateTime.fromISO(sent_at) : null, + new UUID(uuid!) ) server.statsd?.timing('single-ingestion', singleIngestionTimer) }) From 74d667cdf59db02fc3403fff8999eb65b5e196bc Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 16:31:56 +0100 Subject: [PATCH 074/218] Make some fixes and improvements --- src/ingestion/process-event.ts | 128 +++++++++++++-------------------- 1 file changed, 49 insertions(+), 79 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index fc03229e..efa0bddc 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -19,7 +19,7 @@ export class EventsProcessor { this.kafkaProducer = pluginsServer.kafkaProducer! } - async process_event_ee( + public async process_event_ee( distinct_id: string, ip: string, site_url: string, @@ -64,7 +64,7 @@ export class EventsProcessor { } } - handle_timestamp(data: EventData, now: DateTime, sent_at: DateTime | null): DateTime { + private handle_timestamp(data: EventData, now: DateTime, sent_at: DateTime | null): DateTime { if (data['timestamp']) { if (sent_at) { // sent_at - timestamp == now - x @@ -85,7 +85,7 @@ export class EventsProcessor { return now } - async handle_identify_or_alias( + private async handle_identify_or_alias( event: string, properties: Properties, distinct_id: string, @@ -104,41 +104,37 @@ export class EventsProcessor { } } - async _set_is_identified(team_id: number, distinct_id: string, is_identified = true): Promise { - let personFound: Person | undefined - personFound = ( - await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', - [team_id, distinct_id] - ) - ).rows[0] + private async _set_is_identified(team_id: number, distinct_id: string, is_identified = true): Promise { + let personFound = await this.fetchPerson(team_id, distinct_id) if (!personFound) { try { const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, true, new UUIDT()) this.add_distinct_id(personCreated, distinct_id) - // Catch race condition where in between getting and creating, another request already created this person } catch { - personFound = ( - await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', - [team_id, distinct_id] - ) - ).rows[0] + // Catch race condition where in between getting and creating, + // another request already created this person + personFound = await this.fetchPerson(team_id, distinct_id) } } if (personFound && !personFound.is_identified) { - await this.db.query('UPDATE posthog_person SET is_identified = 1 WHERE id = $1', [personFound.id]) + await this.db.query('UPDATE posthog_person SET is_identified = TRUE WHERE id = $1', [personFound.id]) } } - async _update_person_properties(team_id: number, distinct_id: string, properties: Properties): Promise { - let personFound: Person | undefined - personFound = ( - await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', - [team_id, distinct_id] - ) - ).rows[0] + private async fetchPerson(teamId: number, distinctId: string): Promise { + const selectResult = await this.db.query( + 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', + [teamId, distinctId] + ) + return selectResult.rows[0] + } + + private async _update_person_properties( + team_id: number, + distinct_id: string, + properties: Properties + ): Promise { + let personFound = await this.fetchPerson(team_id, distinct_id) if (!personFound) { try { const personCreated = await this.create_person( @@ -150,14 +146,10 @@ export class EventsProcessor { new UUIDT() ) await this.add_distinct_id(personCreated, distinct_id) - // Catch race condition where in between getting and creating, another request already created this person } catch { - personFound = ( - await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', - [team_id, distinct_id] - ) - ).rows[0] + // Catch race condition where in between getting and creating, + // another request already created this person + personFound = await this.fetchPerson(team_id, distinct_id) } } if (personFound) { @@ -168,25 +160,14 @@ export class EventsProcessor { } } - async _alias( + private async _alias( previous_distinct_id: string, distinct_id: string, team_id: number, retry_if_failed = true ): Promise { - const old_person: Person | undefined = ( - await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', - [team_id, previous_distinct_id] - ) - ).rows[0] - - const new_person: Person | undefined = ( - await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', - [team_id, distinct_id] - ) - ).rows[0] + const old_person = await this.fetchPerson(team_id, previous_distinct_id) + const new_person = await this.fetchPerson(team_id, distinct_id) if (old_person && !new_person) { try { @@ -221,11 +202,11 @@ export class EventsProcessor { const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, false, new UUIDT()) this.add_distinct_id(personCreated, distinct_id) this.add_distinct_id(personCreated, previous_distinct_id) - // Catch race condition where in between getting and creating, another request already created this user. } catch { - // integrity error + // Catch race condition where in between getting and creating, + // another request already created this person if (retry_if_failed) { - // try once more, probably one of the two persons exists now + // Try once more, probably one of the two persons exists now this._alias(previous_distinct_id, distinct_id, team_id, false) } } @@ -237,7 +218,7 @@ export class EventsProcessor { } } - async merge_people(merge_into: Person, people_to_merge: Person[]): Promise { + private async merge_people(merge_into: Person, people_to_merge: Person[]): Promise { let first_seen = merge_into.created_at // merge the properties @@ -283,7 +264,7 @@ export class EventsProcessor { } } - async _capture_ee( + private async _capture_ee( event_uuid: UUID, person_uuid: UUID, ip: string, @@ -311,14 +292,7 @@ export class EventsProcessor { })) } - const { - rows: [team], - }: { - rows: Team[] - } = await this.db.query( - 'SELECT slack_incoming_webhook, event_names, event_properties, event_names_with_usage, event_properties_with_usage, anonymize_ips FROM posthog_team WHERE id = $1', - [team_id] - ) + const team: Team = (await this.db.query('SELECT * FROM posthog_team WHERE id = $1', [team_id])).rows[0] if (!team.anonymize_ips && !('$ip' in properties)) { properties['$ip'] = ip @@ -326,14 +300,11 @@ export class EventsProcessor { this.store_names_and_properties(team, event, properties) - const { - rows: [{ pdiCount }], - }: { - rows: { pdiCount: number }[] - } = await this.db.query( - 'SELECT COUNT(*) AS pdiCount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', + const pdiSelectResult = await this.db.query( + 'SELECT COUNT(*) AS pdicount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', [team_id, distinct_id] ) + const pdiCount = parseInt(pdiSelectResult.rows[0].pdicount) if (!pdiCount) { // Catch race condition where in between getting and creating, another request already created this user @@ -353,7 +324,7 @@ export class EventsProcessor { await this.create_event(event_uuid, event, team, distinct_id, properties, timestamp, elements_list) } - async store_names_and_properties(team: Team, event: string, properties: Properties): Promise { + private async store_names_and_properties(team: Team, event: string, properties: Properties): Promise { // In _capture we only prefetch a couple of fields in Team to avoid fetching too much data let save = false if (!team.ingested_event) { @@ -398,7 +369,7 @@ export class EventsProcessor { } } - async create_person( + private async create_person( created_at: DateTime, properties: Properties, team_id: number, @@ -406,22 +377,21 @@ export class EventsProcessor { is_identified: boolean, uuid: UUID | string ): Promise { - return ( - await this.db.query( - 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6)', - [created_at.toISO(), properties, team_id, is_user_id, is_identified, uuid.toString()] - ) - ).rows[0] + const insertResult = await this.db.query( + 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', + [created_at.toISO(), properties, team_id, is_user_id, is_identified, uuid.toString()] + ) + return insertResult.rows[0] } - async add_distinct_id(person: Person, distinct_id: string): Promise { + private async add_distinct_id(person: Person, distinct_id: string): Promise { await this.db.query( - 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3)', + 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', [distinct_id, person.id, person.team_id] ) } - async create_event( + private async create_event( event_uuid: UUID, event: string, team: Team, @@ -453,7 +423,7 @@ export class EventsProcessor { return eventUuidString } - async create_session_recording_event( + private async create_session_recording_event( uuid: UUID, team_id: number, distinct_id: string, From 156039b0d232bac243f5f0a2784daa9029f5a162 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 16:32:15 +0100 Subject: [PATCH 075/218] Remove console.logs --- src/ingestion/process-event.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index efa0bddc..fb35f17e 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -38,7 +38,7 @@ export class EventsProcessor { const ts = this.handle_timestamp(data, now, sent_at) this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) - console.log(`Processing ${data.event}`) + if (data['event'] === '$snapshot') { await this.create_session_recording_event( event_uuid, @@ -414,7 +414,7 @@ export class EventsProcessor { elementsChain, createdAt: timestampString, }) - console.log(`Producing ${event}`) + await this.kafkaProducer.send({ topic: KAFKA_EVENTS, messages: [{ key: eventUuidString, value: EventProto.encodeDelimited(message).finish() as Buffer }], From 81f01ed5e97ecb081f5de063415233055c420125 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 16:38:02 +0100 Subject: [PATCH 076/218] Use plugin-scaffold 0.2.7 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 0ae601a5..86abd034 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "@babel/core": "^7.0.0", "@babel/preset-env": "^7.0.0", "@babel/preset-typescript": "^7.8.3", - "@posthog/plugin-scaffold": "0.2.6", + "@posthog/plugin-scaffold": "0.2.7", "@types/adm-zip": "^0.4.33", "@types/ioredis": "^4.17.7", "@types/jest": "^26.0.15", diff --git a/yarn.lock b/yarn.lock index efa4fd88..1535d3b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1179,10 +1179,10 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" -"@posthog/plugin-scaffold@0.2.6": - version "0.2.6" - resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.6.tgz#5e27067e19ce42a78b67fbbafc001f816ccb0c70" - integrity sha512-W4Ata+n1Zsb+yAVlAQv3OEuHOmwz3Fp6en9iXq5dNNM0gxFoYtKAhzITvz7idAvIy15sa2244E8itSTbqEzzKA== +"@posthog/plugin-scaffold@0.2.7": + version "0.2.7" + resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.7.tgz#0d4a87385ec5c88184ad252a2587b9793f50a58d" + integrity sha512-bgrXLwUMMJ5TzyNK41jlgKdpZffvwjDxYeZ7nL1VDAI/Svv4/DSIA7JtcXZV14q2x99vB3woPckWUUAC7K2jmg== "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" From 153893e18be792661074509d96f05c8718233633 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 17:10:07 +0100 Subject: [PATCH 077/218] Fix style camelCase FTW --- src/ingestion/process-event.ts | 254 +++++++++++++++++---------------- src/worker/queue.ts | 2 +- 2 files changed, 134 insertions(+), 122 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index fb35f17e..bc9c4961 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -19,60 +19,60 @@ export class EventsProcessor { this.kafkaProducer = pluginsServer.kafkaProducer! } - public async process_event_ee( - distinct_id: string, + public async processEventEE( + distinctId: string, ip: string, - site_url: string, + siteUrl: string, data: PluginEvent, - team_id: number, + teamId: number, now: DateTime, - sent_at: DateTime | null, - event_uuid: UUID + sentAt: DateTime | null, + eventUuid: UUID ): Promise { const properties: Properties = data.properties ?? {} if (data['$set']) { properties['$set'] = data['$set'] } - const person_uuid = new UUIDT() + const personUuid = new UUIDT() - const ts = this.handle_timestamp(data, now, sent_at) - this.handle_identify_or_alias(data['event'], properties, distinct_id, team_id) + const ts = this.handleTimestamp(data, now, sentAt) + this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) if (data['event'] === '$snapshot') { - await this.create_session_recording_event( - event_uuid, - team_id, - distinct_id, + await this.createSessionRecordingEvent( + eventUuid, + teamId, + distinctId, properties['$session_id'], ts, properties['$snapshot_data'] ) } else { - await this._capture_ee( - event_uuid, - person_uuid, + await this.captureEE( + eventUuid, + personUuid, ip, - site_url, - team_id, + siteUrl, + teamId, data['event'], - distinct_id, + distinctId, properties, ts, - sent_at + sentAt ) } } - private handle_timestamp(data: EventData, now: DateTime, sent_at: DateTime | null): DateTime { + private handleTimestamp(data: EventData, now: DateTime, sentAt: DateTime | null): DateTime { if (data['timestamp']) { - if (sent_at) { + if (sentAt) { // sent_at - timestamp == now - x // x = now + (timestamp - sent_at) try { // timestamp and sent_at must both be in the same format: either both with or both without timezones // otherwise we can't get a diff to add to now - return now.plus(DateTime.fromISO(data['timestamp']).diff(sent_at)) + return now.plus(DateTime.fromISO(data['timestamp']).diff(sentAt)) } catch (error) { console.error(error) } @@ -85,71 +85,62 @@ export class EventsProcessor { return now } - private async handle_identify_or_alias( + private async handleIdentifyOrAlias( event: string, properties: Properties, - distinct_id: string, - team_id: number + distinctId: string, + teamId: number ): Promise { if (event === '$create_alias') { - await this._alias(properties['alias'], distinct_id, team_id) + await this.alias(properties['alias'], distinctId, teamId) } else if (event === '$identify') { if (properties['$anon_distinct_id']) { - await this._alias(properties['$anon_distinct_id'], distinct_id, team_id) + await this.alias(properties['$anon_distinct_id'], distinctId, teamId) } if (properties['$set']) { - this._update_person_properties(team_id, distinct_id, properties['$set']) + this.updatePersonProperties(teamId, distinctId, properties['$set']) } - this._set_is_identified(team_id, distinct_id) + this.setIsIdentified(teamId, distinctId) } } - private async _set_is_identified(team_id: number, distinct_id: string, is_identified = true): Promise { - let personFound = await this.fetchPerson(team_id, distinct_id) + private async setIsIdentified(teamId: number, distinctId: string, isIdentified = true): Promise { + let personFound = await this.fetchPerson(teamId, distinctId) if (!personFound) { try { - const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, true, new UUIDT()) - this.add_distinct_id(personCreated, distinct_id) + const personCreated = await this.createPerson(DateTime.utc(), {}, teamId, null, true, new UUIDT()) + this.addDistinctId(personCreated, distinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person - personFound = await this.fetchPerson(team_id, distinct_id) + personFound = await this.fetchPerson(teamId, distinctId) } } if (personFound && !personFound.is_identified) { - await this.db.query('UPDATE posthog_person SET is_identified = TRUE WHERE id = $1', [personFound.id]) + await this.db.query('UPDATE posthog_person SET is_identified = $1 WHERE id = $2', [ + isIdentified, + personFound.id, + ]) } } - private async fetchPerson(teamId: number, distinctId: string): Promise { - const selectResult = await this.db.query( - 'SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, posthog_persondistinctid.team_id AS persondistinctid__team_id, posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id FROM posthog_person JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) WHERE posthog_person.team_id = $1 AND posthog_persondistinctid.team_id = $1 AND posthog_persondistinctid.distinct_id = $2', - [teamId, distinctId] - ) - return selectResult.rows[0] - } - - private async _update_person_properties( - team_id: number, - distinct_id: string, - properties: Properties - ): Promise { - let personFound = await this.fetchPerson(team_id, distinct_id) + private async updatePersonProperties(teamId: number, distinctId: string, properties: Properties): Promise { + let personFound = await this.fetchPerson(teamId, distinctId) if (!personFound) { try { - const personCreated = await this.create_person( + const personCreated = await this.createPerson( DateTime.utc(), properties, - team_id, + teamId, null, false, new UUIDT() ) - await this.add_distinct_id(personCreated, distinct_id) + await this.addDistinctId(personCreated, distinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person - personFound = await this.fetchPerson(team_id, distinct_id) + personFound = await this.fetchPerson(teamId, distinctId) } } if (personFound) { @@ -160,70 +151,70 @@ export class EventsProcessor { } } - private async _alias( - previous_distinct_id: string, - distinct_id: string, - team_id: number, - retry_if_failed = true + private async alias( + previousDistinctId: string, + distinctId: string, + teamId: number, + retryIfFailed = true ): Promise { - const old_person = await this.fetchPerson(team_id, previous_distinct_id) - const new_person = await this.fetchPerson(team_id, distinct_id) + const oldPerson = await this.fetchPerson(teamId, previousDistinctId) + const newPerson = await this.fetchPerson(teamId, distinctId) - if (old_person && !new_person) { + if (oldPerson && !newPerson) { try { - this.add_distinct_id(old_person, distinct_id) - // Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + this.addDistinctId(oldPerson, distinctId) + // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error - if (retry_if_failed) { + if (retryIfFailed) { // run everything again to merge the users if needed - this._alias(previous_distinct_id, distinct_id, team_id, false) + this.alias(previousDistinctId, distinctId, teamId, false) } } return } - if (!old_person && new_person) { + if (!oldPerson && newPerson) { try { - this.add_distinct_id(new_person, previous_distinct_id) - // Catch race case when somebody already added this distinct_id between .get and .add_distinct_id + this.addDistinctId(newPerson, previousDistinctId) + // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error - if (retry_if_failed) { + if (retryIfFailed) { // run everything again to merge the users if needed - this._alias(previous_distinct_id, distinct_id, team_id, false) + this.alias(previousDistinctId, distinctId, teamId, false) } } return } - if (!old_person && !new_person) { + if (!oldPerson && !newPerson) { try { - const personCreated = await this.create_person(DateTime.utc(), {}, team_id, null, false, new UUIDT()) - this.add_distinct_id(personCreated, distinct_id) - this.add_distinct_id(personCreated, previous_distinct_id) + const personCreated = await this.createPerson(DateTime.utc(), {}, teamId, null, false, new UUIDT()) + this.addDistinctId(personCreated, distinctId) + this.addDistinctId(personCreated, previousDistinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person - if (retry_if_failed) { + if (retryIfFailed) { // Try once more, probably one of the two persons exists now - this._alias(previous_distinct_id, distinct_id, team_id, false) + this.alias(previousDistinctId, distinctId, teamId, false) } } return } - if (old_person && new_person && old_person.id !== new_person.id) { - this.merge_people(new_person, [old_person]) + if (oldPerson && newPerson && oldPerson.id !== newPerson.id) { + this.mergePeople(newPerson, [oldPerson]) } } - private async merge_people(merge_into: Person, people_to_merge: Person[]): Promise { - let first_seen = merge_into.created_at + private async mergePeople(mergeInto: Person, peopleToMerge: Person[]): Promise { + let first_seen = mergeInto.created_at // merge the properties - for (const other_person of people_to_merge) { - merge_into.properties = { ...other_person.properties, ...merge_into.properties } + for (const other_person of peopleToMerge) { + mergeInto.properties = { ...other_person.properties, ...mergeInto.properties } if (other_person.created_at < first_seen) { // Keep the oldest created_at (i.e. the first time we've seen this person) first_seen = other_person.created_at @@ -232,20 +223,20 @@ export class EventsProcessor { await this.db.query('UPDATE posthog_person SET created_at = $1 WHERE id = $2', [ first_seen.toISO(), - merge_into.id, + mergeInto.id, ]) // merge the distinct_ids - for (const other_person of people_to_merge) { + for (const other_person of peopleToMerge) { const other_person_distinct_ids: PersonDistinctId[] = ( await this.db.query('SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', [ other_person, - merge_into.team_id, + mergeInto.team_id, ]) ).rows for (const person_distinct_id of other_person_distinct_ids) { await this.db.query('UPDATE posthog_persondistinctid SET person_id = $1 WHERE id = $2', [ - merge_into.id, + mergeInto.id, person_distinct_id.id, ]) } @@ -255,7 +246,7 @@ export class EventsProcessor { ).rows for (const person_cohort_id of other_person_cohort_ids) { await this.db.query('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ - merge_into.id, + mergeInto.id, person_cohort_id.id, ]) } @@ -264,17 +255,17 @@ export class EventsProcessor { } } - private async _capture_ee( - event_uuid: UUID, - person_uuid: UUID, + private async captureEE( + eventUuid: UUID, + personUuid: UUID, ip: string, - site_url: string, - team_id: number, + siteUrl: string, + teamId: number, event: string, - distinct_id: string, + distinctId: string, properties: Properties, timestamp: DateTime, - sent_at: DateTime | null + sentAt: DateTime | null ): Promise { const elements: Record[] | undefined = properties['$elements'] let elements_list: Element[] = [] @@ -292,39 +283,39 @@ export class EventsProcessor { })) } - const team: Team = (await this.db.query('SELECT * FROM posthog_team WHERE id = $1', [team_id])).rows[0] + const team: Team = (await this.db.query('SELECT * FROM posthog_team WHERE id = $1', [teamId])).rows[0] if (!team.anonymize_ips && !('$ip' in properties)) { properties['$ip'] = ip } - this.store_names_and_properties(team, event, properties) + this.storeNamesAndProperties(team, event, properties) const pdiSelectResult = await this.db.query( 'SELECT COUNT(*) AS pdicount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', - [team_id, distinct_id] + [teamId, distinctId] ) const pdiCount = parseInt(pdiSelectResult.rows[0].pdicount) if (!pdiCount) { // Catch race condition where in between getting and creating, another request already created this user try { - const personCreated: Person = await this.create_person( - sent_at || DateTime.utc(), + const personCreated: Person = await this.createPerson( + sentAt || DateTime.utc(), {}, - team_id, + teamId, null, false, - person_uuid.toString() + personUuid.toString() ) - await this.add_distinct_id(personCreated, distinct_id) + await this.addDistinctId(personCreated, distinctId) } catch {} } - await this.create_event(event_uuid, event, team, distinct_id, properties, timestamp, elements_list) + await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elements_list) } - private async store_names_and_properties(team: Team, event: string, properties: Properties): Promise { + private async storeNamesAndProperties(team: Team, event: string, properties: Properties): Promise { // In _capture we only prefetch a couple of fields in Team to avoid fetching too much data let save = false if (!team.ingested_event) { @@ -355,7 +346,10 @@ export class EventsProcessor { } if (save) { await this.db.query( - 'UPDATE posthog_team SET ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, event_properties_with_usage = $5, event_properties_numerical = $6 WHERE id = $7', + `UPDATE posthog_team SET + ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, + event_properties_with_usage = $5, event_properties_numerical = $6 + WHERE id = $7`, [ team.ingested_event, JSON.stringify(team.event_names), @@ -369,29 +363,47 @@ export class EventsProcessor { } } - private async create_person( - created_at: DateTime, + private async fetchPerson(teamId: number, distinctId: string): Promise { + const selectResult = await this.db.query( + `SELECT + posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, + posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, + posthog_persondistinctid.team_id AS persondistinctid__team_id, + posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id + FROM posthog_person + JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) + WHERE + posthog_person.team_id = $1 + AND posthog_persondistinctid.team_id = $1 + AND posthog_persondistinctid.distinct_id = $2`, + [teamId, distinctId] + ) + return selectResult.rows[0] + } + + private async createPerson( + createdAt: DateTime, properties: Properties, - team_id: number, - is_user_id: number | null, - is_identified: boolean, + teamId: number, + isUserId: number | null, + isIdentified: boolean, uuid: UUID | string ): Promise { const insertResult = await this.db.query( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', - [created_at.toISO(), properties, team_id, is_user_id, is_identified, uuid.toString()] + [createdAt.toISO(), properties, teamId, isUserId, isIdentified, uuid.toString()] ) return insertResult.rows[0] } - private async add_distinct_id(person: Person, distinct_id: string): Promise { + private async addDistinctId(person: Person, distinctId: string): Promise { await this.db.query( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', - [distinct_id, person.id, person.team_id] + [distinctId, person.id, person.team_id] ) } - private async create_event( + private async createEvent( event_uuid: UUID, event: string, team: Team, @@ -423,8 +435,8 @@ export class EventsProcessor { return eventUuidString } - private async create_session_recording_event( - uuid: UUID, + private async createSessionRecordingEvent( + eventUuid: UUID, team_id: number, distinct_id: string, session_id: string, @@ -432,10 +444,10 @@ export class EventsProcessor { snapshot_data: Record ): Promise { const timestampString = castTimestampOrNow(timestamp) - const uuidString = uuid.toString() + const eventUuidString = eventUuid.toString() const data = { - uuid: uuidString, + uuid: eventUuidString, team_id: team_id, distinct_id: distinct_id, session_id: session_id, @@ -446,9 +458,9 @@ export class EventsProcessor { await this.kafkaProducer.send({ topic: KAFKA_SESSION_RECORDING_EVENTS, - messages: [{ key: uuidString, value: Buffer.from(JSON.stringify(data)) }], + messages: [{ key: eventUuidString, value: Buffer.from(JSON.stringify(data)) }], }) - return uuidString + return eventUuidString } } diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 6f9f3a6e..a42d47b9 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -69,7 +69,7 @@ function startQueueKafka( const kafkaQueue = new KafkaQueue(server, processEventBatch, async (event: PluginEvent) => { const singleIngestionTimer = new Date() const { distinct_id, ip, site_url, team_id, now, sent_at, uuid } = event - await server.eventsProcessor.process_event_ee( + await server.eventsProcessor.processEventEE( distinct_id, ip, site_url, From e5b5892f285a5c420490cd9b26b373a70fb124f5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 17:14:04 +0100 Subject: [PATCH 078/218] Simplify compile:typescript --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 86abd034..85e14e22 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && eslint --fix . && prettier --write .", - "compile:typescript": "tsc -p .", + "compile:typescript": "tsc", "compile": "yarn compile:protobuf && yarn compile:typescript", "lint": "eslint .", "lint:fix": "eslint --fix .", From 1ca29ba2c6db22dceef4fb02dac5136846d4dfa1 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 17:34:52 +0100 Subject: [PATCH 079/218] await startQueue --- src/server.ts | 4 ++-- src/worker/queue.ts | 18 +++++++++--------- tests/queue.test.ts | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/server.ts b/src/server.ts index cc5eeac1..9fc9157c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -208,7 +208,7 @@ export async function startPluginsServer( fastifyInstance = await startFastifyInstance(server) } - queue = startQueue(server, processEvent, processEventBatch) + queue = await startQueue(server, processEvent, processEventBatch) piscina.on('drain', () => { queue?.resume() }) @@ -222,7 +222,7 @@ export async function startPluginsServer( await stopSchedule?.() await stopPiscina(piscina!) piscina = makePiscina(serverConfig!) - queue = startQueue(server!, processEvent, processEventBatch) + queue = await startQueue(server!, processEvent, processEventBatch) stopSchedule = await startSchedule(server!, piscina) } }) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index a42d47b9..89bdd5de 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -7,20 +7,20 @@ import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' import { UUID } from '../utils' -export function startQueue( +export async function startQueue( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Queue { +): Promise { const relevantStartQueue = server.KAFKA_ENABLED ? startQueueKafka : startQueueRedis - return relevantStartQueue(server, processEvent, processEventBatch) + return await relevantStartQueue(server, processEvent, processEventBatch) } -function startQueueRedis( +async function startQueueRedis( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Queue { +): Promise { const worker = new Worker(server.redis, server.PLUGINS_CELERY_QUEUE) const client = new Client(server.redis, server.CELERY_DEFAULT_QUEUE) @@ -56,16 +56,16 @@ function startQueueRedis( } ) - worker.start() + await worker.start() return worker } -function startQueueKafka( +async function startQueueKafka( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Queue { +): Promise { const kafkaQueue = new KafkaQueue(server, processEventBatch, async (event: PluginEvent) => { const singleIngestionTimer = new Date() const { distinct_id, ip, site_url, team_id, now, sent_at, uuid } = event @@ -82,7 +82,7 @@ function startQueueKafka( server.statsd?.timing('single-ingestion', singleIngestionTimer) }) - kafkaQueue.start() + await kafkaQueue.start() return kafkaQueue } diff --git a/tests/queue.test.ts b/tests/queue.test.ts index f79824b8..78b003d8 100644 --- a/tests/queue.test.ts +++ b/tests/queue.test.ts @@ -67,7 +67,7 @@ test('worker and task passing via redis', async () => { expect(args2).toEqual(args) expect(kwargs2).toEqual({}) - const queue = startQueue( + const queue = await startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) @@ -132,7 +132,7 @@ test('process multiple tasks', async () => { expect(await server.redis.llen(server.PLUGINS_CELERY_QUEUE)).toBe(3) expect(await server.redis.llen(server.CELERY_DEFAULT_QUEUE)).toBe(0) - const queue = startQueue( + const queue = await startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) @@ -199,7 +199,7 @@ test('pause and resume queue', async () => { expect(await server.redis.llen(server.PLUGINS_CELERY_QUEUE)).toBe(6) expect(await server.redis.llen(server.CELERY_DEFAULT_QUEUE)).toBe(0) - const queue = startQueue( + const queue = await startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) From 81846ef569f45c3c33594eafb2ba541e520ae336 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 17:52:17 +0100 Subject: [PATCH 080/218] Clean up castTimestampOrNow --- src/utils.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 09ac63f6..53216d2f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -267,13 +267,9 @@ export class UUIDT extends UUID { } export function castTimestampOrNow(timestamp?: DateTime | string | null): string { - if (!timestamp) { - timestamp = DateTime.utc() - } - - // ClickHouse specific formatting + timestamp ||= DateTime.utc() + // ClickHouse-specific formatting timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() - return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') } From 9a03a885243b0646b7ee49b88bc088e24188c4bb Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 12 Jan 2021 18:05:40 +0100 Subject: [PATCH 081/218] Don't use pesky ||= --- src/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 53216d2f..27dd3206 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -267,7 +267,9 @@ export class UUIDT extends UUID { } export function castTimestampOrNow(timestamp?: DateTime | string | null): string { - timestamp ||= DateTime.utc() + if (!timestamp) { + timestamp = DateTime.utc() + } // ClickHouse-specific formatting timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') From 4b3f3fdefd3c2e517a3a15235253a5a00d9cc007 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 13 Jan 2021 03:12:43 +0100 Subject: [PATCH 082/218] Change consumer group to 'group1' from main repo --- src/ingestion/kafka-queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 93d90473..8c26a598 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -113,7 +113,7 @@ export class KafkaQueue implements Queue { private static buildConsumer(kafka: Kafka): Consumer { const consumer = kafka.consumer({ - groupId: 'plugin-server', + groupId: 'group1', readUncommitted: false, }) const { GROUP_JOIN, CRASH, CONNECT, DISCONNECT } = consumer.events From 6d20d34b5c295b736ade4858d9a4b882cf4fd970 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 13 Jan 2021 03:13:41 +0100 Subject: [PATCH 083/218] Revert "await startQueue" This reverts commit 1ca29ba2c6db22dceef4fb02dac5136846d4dfa1. --- src/server.ts | 4 ++-- src/worker/queue.ts | 18 +++++++++--------- tests/queue.test.ts | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/server.ts b/src/server.ts index 9fc9157c..cc5eeac1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -208,7 +208,7 @@ export async function startPluginsServer( fastifyInstance = await startFastifyInstance(server) } - queue = await startQueue(server, processEvent, processEventBatch) + queue = startQueue(server, processEvent, processEventBatch) piscina.on('drain', () => { queue?.resume() }) @@ -222,7 +222,7 @@ export async function startPluginsServer( await stopSchedule?.() await stopPiscina(piscina!) piscina = makePiscina(serverConfig!) - queue = await startQueue(server!, processEvent, processEventBatch) + queue = startQueue(server!, processEvent, processEventBatch) stopSchedule = await startSchedule(server!, piscina) } }) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 89bdd5de..a42d47b9 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -7,20 +7,20 @@ import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' import { UUID } from '../utils' -export async function startQueue( +export function startQueue( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Promise { +): Queue { const relevantStartQueue = server.KAFKA_ENABLED ? startQueueKafka : startQueueRedis - return await relevantStartQueue(server, processEvent, processEventBatch) + return relevantStartQueue(server, processEvent, processEventBatch) } -async function startQueueRedis( +function startQueueRedis( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Promise { +): Queue { const worker = new Worker(server.redis, server.PLUGINS_CELERY_QUEUE) const client = new Client(server.redis, server.CELERY_DEFAULT_QUEUE) @@ -56,16 +56,16 @@ async function startQueueRedis( } ) - await worker.start() + worker.start() return worker } -async function startQueueKafka( +function startQueueKafka( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Promise { +): Queue { const kafkaQueue = new KafkaQueue(server, processEventBatch, async (event: PluginEvent) => { const singleIngestionTimer = new Date() const { distinct_id, ip, site_url, team_id, now, sent_at, uuid } = event @@ -82,7 +82,7 @@ async function startQueueKafka( server.statsd?.timing('single-ingestion', singleIngestionTimer) }) - await kafkaQueue.start() + kafkaQueue.start() return kafkaQueue } diff --git a/tests/queue.test.ts b/tests/queue.test.ts index 78b003d8..f79824b8 100644 --- a/tests/queue.test.ts +++ b/tests/queue.test.ts @@ -67,7 +67,7 @@ test('worker and task passing via redis', async () => { expect(args2).toEqual(args) expect(kwargs2).toEqual({}) - const queue = await startQueue( + const queue = startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) @@ -132,7 +132,7 @@ test('process multiple tasks', async () => { expect(await server.redis.llen(server.PLUGINS_CELERY_QUEUE)).toBe(3) expect(await server.redis.llen(server.CELERY_DEFAULT_QUEUE)).toBe(0) - const queue = await startQueue( + const queue = startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) @@ -199,7 +199,7 @@ test('pause and resume queue', async () => { expect(await server.redis.llen(server.PLUGINS_CELERY_QUEUE)).toBe(6) expect(await server.redis.llen(server.CELERY_DEFAULT_QUEUE)).toBe(0) - const queue = await startQueue( + const queue = startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) From 26b5f4451eb066ae0d4523fe9feefa485ace995b Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 13 Jan 2021 03:17:02 +0100 Subject: [PATCH 084/218] Exit if queue failed to start --- src/worker/queue.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index a42d47b9..f5d53ed8 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -6,6 +6,7 @@ import Client from '../celery/client' import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' import { UUID } from '../utils' +import { status } from '../status' export function startQueue( server: PluginsServer, @@ -56,7 +57,10 @@ function startQueueRedis( } ) - worker.start() + worker.start().catch((reason) => { + status.error('๐Ÿ’ฅ', `Failed to start Redis queue:\n${reason}`) + process.exit(1) + }) return worker } @@ -82,7 +86,10 @@ function startQueueKafka( server.statsd?.timing('single-ingestion', singleIngestionTimer) }) - kafkaQueue.start() + kafkaQueue.start().catch((reason) => { + status.error('๐Ÿ’ฅ', `Failed to start Kafka queue:\n${reason}`) + process.exit(1) + }) return kafkaQueue } From 6dce496692aa15e68af5955a42e789de526b112d Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 13 Jan 2021 11:53:43 +0100 Subject: [PATCH 085/218] Clean up elements handling --- src/ingestion/element.ts | 6 +++--- src/ingestion/process-event.ts | 4 ++-- src/types.ts | 19 +++++++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/ingestion/element.ts b/src/ingestion/element.ts index af620f79..d17deb1f 100644 --- a/src/ingestion/element.ts +++ b/src/ingestion/element.ts @@ -1,10 +1,10 @@ import { Element } from 'types' -function _escape(input: string): string { +function escapeQuotes(input: string): string { return input.replace(/"/g, '\\"') } -export function elements_to_string(elements: Element[]): string { +export function elementsToString(elements: Element[]): string { const ret = elements.map((element) => { let el_string = '' if (element.tag_name) { @@ -27,7 +27,7 @@ export function elements_to_string(elements: Element[]): string { attributes = Object.fromEntries( Object.entries(attributes) .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, value]) => [_escape(key.toString()), _escape(value.toString())]) + .map(([key, value]) => [escapeQuotes(key.toString()), escapeQuotes(value.toString())]) ) el_string += ':' el_string += Object.entries(attributes) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index bc9c4961..0ff67221 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -2,7 +2,7 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { DateTime, Duration } from 'luxon' import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' import { castTimestampOrNow, UUID, UUIDT } from '../utils' -import { elements_to_string } from './element' +import { elementsToString } from './element' import { Event as EventProto } from '../idl/protos' import { Pool } from 'pg' import { Producer } from 'kafkajs' @@ -413,7 +413,7 @@ export class EventsProcessor { elements?: Element[] ): Promise { const timestampString = castTimestampOrNow(timestamp) - const elementsChain = elements && elements.length ? elements_to_string(elements) : '' + const elementsChain = elements && elements.length ? elementsToString(elements) : '' const eventUuidString = event_uuid.toString() const message = EventProto.create({ diff --git a/src/types.ts b/src/types.ts index d399fe7c..71cc2268 100644 --- a/src/types.ts +++ b/src/types.ts @@ -211,14 +211,17 @@ export interface Team { } export interface Element { - text: string - tag_name: string - href: string - attr_class: string[] - attr_id: string - nth_child: number - nth_of_type: number - attributes: Record + text?: string + tag_name?: string + href?: string + attr_id?: string + attr_class?: string[] + nth_child?: number + nth_of_type?: number + attributes: Record + event_id?: number + order?: number + group_id?: number } export type User = Record // not really typed as not needed so far, only a placeholder for Person.is_user From 3680481923de37123cdd2c53cec0b68bb159c9ef Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 13 Jan 2021 12:01:35 +0100 Subject: [PATCH 086/218] Don't commit compiled protos --- .gitignore | 1 + package.json | 1 + src/idl/protos.d.ts | 248 -------------- src/idl/protos.js | 787 -------------------------------------------- 4 files changed, 2 insertions(+), 1035 deletions(-) delete mode 100644 src/idl/protos.d.ts delete mode 100644 src/idl/protos.js diff --git a/.gitignore b/.gitignore index 8aa1a5b5..3f6480bc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ yarn-error.log dist/ .yalc/ yalc.lock +src/idl/protos.* diff --git a/package.json b/package.json index 561ff2bd..4803910e 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "lint:fix": "eslint --fix .", "prettier": "prettier --write .", "prettier:check": "prettier --check .", + "prepare": "yarn compile:protobuf", "prepublishOnly": "yarn test && yarn build", "task:init-test-db": "NODE_ENV=test ts-node ./tasks/init-test-db.ts" }, diff --git a/src/idl/protos.d.ts b/src/idl/protos.d.ts deleted file mode 100644 index bbf095d0..00000000 --- a/src/idl/protos.d.ts +++ /dev/null @@ -1,248 +0,0 @@ -import * as $protobuf from 'protobufjs' -/** Properties of an Event. */ -export interface IEvent { - /** Event uuid */ - uuid?: string | null - - /** Event event */ - event?: string | null - - /** Event properties */ - properties?: string | null - - /** Event timestamp */ - timestamp?: string | null - - /** Event teamId */ - teamId?: number | Long | null - - /** Event distinctId */ - distinctId?: string | null - - /** Event createdAt */ - createdAt?: string | null - - /** Event elementsChain */ - elementsChain?: string | null - - /** Event protoCreatedAt */ - protoCreatedAt?: google.protobuf.ITimestamp | null - - /** Event protoTimestamp */ - protoTimestamp?: google.protobuf.ITimestamp | null -} - -/** Represents an Event. */ -export class Event implements IEvent { - /** - * Constructs a new Event. - * @param [properties] Properties to set - */ - constructor(properties?: IEvent) - - /** Event uuid. */ - public uuid: string - - /** Event event. */ - public event: string - - /** Event properties. */ - public properties: string - - /** Event timestamp. */ - public timestamp: string - - /** Event teamId. */ - public teamId: number | Long - - /** Event distinctId. */ - public distinctId: string - - /** Event createdAt. */ - public createdAt: string - - /** Event elementsChain. */ - public elementsChain: string - - /** Event protoCreatedAt. */ - public protoCreatedAt?: google.protobuf.ITimestamp | null - - /** Event protoTimestamp. */ - public protoTimestamp?: google.protobuf.ITimestamp | null - - /** - * Creates a new Event instance using the specified properties. - * @param [properties] Properties to set - * @returns Event instance - */ - public static create(properties?: IEvent): Event - - /** - * Encodes the specified Event message. Does not implicitly {@link Event.verify|verify} messages. - * @param message Event message or plain object to encode - * @param [writer] Writer to encode to - * @returns Writer - */ - public static encode(message: IEvent, writer?: $protobuf.Writer): $protobuf.Writer - - /** - * Encodes the specified Event message, length delimited. Does not implicitly {@link Event.verify|verify} messages. - * @param message Event message or plain object to encode - * @param [writer] Writer to encode to - * @returns Writer - */ - public static encodeDelimited(message: IEvent, writer?: $protobuf.Writer): $protobuf.Writer - - /** - * Decodes an Event message from the specified reader or buffer. - * @param reader Reader or buffer to decode from - * @param [length] Message length if known beforehand - * @returns Event - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - public static decode(reader: $protobuf.Reader | Uint8Array, length?: number): Event - - /** - * Decodes an Event message from the specified reader or buffer, length delimited. - * @param reader Reader or buffer to decode from - * @returns Event - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - public static decodeDelimited(reader: $protobuf.Reader | Uint8Array): Event - - /** - * Verifies an Event message. - * @param message Plain object to verify - * @returns `null` if valid, otherwise the reason why it is not - */ - public static verify(message: { [k: string]: any }): string | null - - /** - * Creates an Event message from a plain object. Also converts values to their respective internal types. - * @param object Plain object - * @returns Event - */ - public static fromObject(object: { [k: string]: any }): Event - - /** - * Creates a plain object from an Event message. Also converts values to other types if specified. - * @param message Event - * @param [options] Conversion options - * @returns Plain object - */ - public static toObject(message: Event, options?: $protobuf.IConversionOptions): { [k: string]: any } - - /** - * Converts this Event to JSON. - * @returns JSON object - */ - public toJSON(): { [k: string]: any } -} - -/** Namespace google. */ -export namespace google { - /** Namespace protobuf. */ - namespace protobuf { - /** Properties of a Timestamp. */ - interface ITimestamp { - /** Timestamp seconds */ - seconds?: number | Long | null - - /** Timestamp nanos */ - nanos?: number | null - } - - /** Represents a Timestamp. */ - class Timestamp implements ITimestamp { - /** - * Constructs a new Timestamp. - * @param [properties] Properties to set - */ - constructor(properties?: google.protobuf.ITimestamp) - - /** Timestamp seconds. */ - public seconds: number | Long - - /** Timestamp nanos. */ - public nanos: number - - /** - * Creates a new Timestamp instance using the specified properties. - * @param [properties] Properties to set - * @returns Timestamp instance - */ - public static create(properties?: google.protobuf.ITimestamp): google.protobuf.Timestamp - - /** - * Encodes the specified Timestamp message. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. - * @param message Timestamp message or plain object to encode - * @param [writer] Writer to encode to - * @returns Writer - */ - public static encode(message: google.protobuf.ITimestamp, writer?: $protobuf.Writer): $protobuf.Writer - - /** - * Encodes the specified Timestamp message, length delimited. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. - * @param message Timestamp message or plain object to encode - * @param [writer] Writer to encode to - * @returns Writer - */ - public static encodeDelimited( - message: google.protobuf.ITimestamp, - writer?: $protobuf.Writer - ): $protobuf.Writer - - /** - * Decodes a Timestamp message from the specified reader or buffer. - * @param reader Reader or buffer to decode from - * @param [length] Message length if known beforehand - * @returns Timestamp - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - public static decode(reader: $protobuf.Reader | Uint8Array, length?: number): google.protobuf.Timestamp - - /** - * Decodes a Timestamp message from the specified reader or buffer, length delimited. - * @param reader Reader or buffer to decode from - * @returns Timestamp - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - public static decodeDelimited(reader: $protobuf.Reader | Uint8Array): google.protobuf.Timestamp - - /** - * Verifies a Timestamp message. - * @param message Plain object to verify - * @returns `null` if valid, otherwise the reason why it is not - */ - public static verify(message: { [k: string]: any }): string | null - - /** - * Creates a Timestamp message from a plain object. Also converts values to their respective internal types. - * @param object Plain object - * @returns Timestamp - */ - public static fromObject(object: { [k: string]: any }): google.protobuf.Timestamp - - /** - * Creates a plain object from a Timestamp message. Also converts values to other types if specified. - * @param message Timestamp - * @param [options] Conversion options - * @returns Plain object - */ - public static toObject( - message: google.protobuf.Timestamp, - options?: $protobuf.IConversionOptions - ): { [k: string]: any } - - /** - * Converts this Timestamp to JSON. - * @returns JSON object - */ - public toJSON(): { [k: string]: any } - } - } -} diff --git a/src/idl/protos.js b/src/idl/protos.js deleted file mode 100644 index 7f96f73e..00000000 --- a/src/idl/protos.js +++ /dev/null @@ -1,787 +0,0 @@ -/*eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars*/ -'use strict' - -var $protobuf = require('protobufjs/minimal') - -// Common aliases -var $Reader = $protobuf.Reader, - $Writer = $protobuf.Writer, - $util = $protobuf.util - -// Exported root namespace -var $root = $protobuf.roots['default'] || ($protobuf.roots['default'] = {}) - -$root.Event = (function () { - /** - * Properties of an Event. - * @exports IEvent - * @interface IEvent - * @property {string|null} [uuid] Event uuid - * @property {string|null} [event] Event event - * @property {string|null} [properties] Event properties - * @property {string|null} [timestamp] Event timestamp - * @property {number|Long|null} [teamId] Event teamId - * @property {string|null} [distinctId] Event distinctId - * @property {string|null} [createdAt] Event createdAt - * @property {string|null} [elementsChain] Event elementsChain - * @property {google.protobuf.ITimestamp|null} [protoCreatedAt] Event protoCreatedAt - * @property {google.protobuf.ITimestamp|null} [protoTimestamp] Event protoTimestamp - */ - - /** - * Constructs a new Event. - * @exports Event - * @classdesc Represents an Event. - * @implements IEvent - * @constructor - * @param {IEvent=} [properties] Properties to set - */ - function Event(properties) { - if (properties) { - for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) { - if (properties[keys[i]] != null) { - this[keys[i]] = properties[keys[i]] - } - } - } - } - - /** - * Event uuid. - * @member {string} uuid - * @memberof Event - * @instance - */ - Event.prototype.uuid = '' - - /** - * Event event. - * @member {string} event - * @memberof Event - * @instance - */ - Event.prototype.event = '' - - /** - * Event properties. - * @member {string} properties - * @memberof Event - * @instance - */ - Event.prototype.properties = '' - - /** - * Event timestamp. - * @member {string} timestamp - * @memberof Event - * @instance - */ - Event.prototype.timestamp = '' - - /** - * Event teamId. - * @member {number|Long} teamId - * @memberof Event - * @instance - */ - Event.prototype.teamId = $util.Long ? $util.Long.fromBits(0, 0, true) : 0 - - /** - * Event distinctId. - * @member {string} distinctId - * @memberof Event - * @instance - */ - Event.prototype.distinctId = '' - - /** - * Event createdAt. - * @member {string} createdAt - * @memberof Event - * @instance - */ - Event.prototype.createdAt = '' - - /** - * Event elementsChain. - * @member {string} elementsChain - * @memberof Event - * @instance - */ - Event.prototype.elementsChain = '' - - /** - * Event protoCreatedAt. - * @member {google.protobuf.ITimestamp|null|undefined} protoCreatedAt - * @memberof Event - * @instance - */ - Event.prototype.protoCreatedAt = null - - /** - * Event protoTimestamp. - * @member {google.protobuf.ITimestamp|null|undefined} protoTimestamp - * @memberof Event - * @instance - */ - Event.prototype.protoTimestamp = null - - /** - * Creates a new Event instance using the specified properties. - * @function create - * @memberof Event - * @static - * @param {IEvent=} [properties] Properties to set - * @returns {Event} Event instance - */ - Event.create = function create(properties) { - return new Event(properties) - } - - /** - * Encodes the specified Event message. Does not implicitly {@link Event.verify|verify} messages. - * @function encode - * @memberof Event - * @static - * @param {IEvent} message Event message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Event.encode = function encode(message, writer) { - if (!writer) { - writer = $Writer.create() - } - if (message.uuid != null && Object.hasOwnProperty.call(message, 'uuid')) { - writer.uint32(/* id 1, wireType 2 =*/ 10).string(message.uuid) - } - if (message.event != null && Object.hasOwnProperty.call(message, 'event')) { - writer.uint32(/* id 2, wireType 2 =*/ 18).string(message.event) - } - if (message.properties != null && Object.hasOwnProperty.call(message, 'properties')) { - writer.uint32(/* id 3, wireType 2 =*/ 26).string(message.properties) - } - if (message.timestamp != null && Object.hasOwnProperty.call(message, 'timestamp')) { - writer.uint32(/* id 4, wireType 2 =*/ 34).string(message.timestamp) - } - if (message.teamId != null && Object.hasOwnProperty.call(message, 'teamId')) { - writer.uint32(/* id 5, wireType 0 =*/ 40).uint64(message.teamId) - } - if (message.distinctId != null && Object.hasOwnProperty.call(message, 'distinctId')) { - writer.uint32(/* id 6, wireType 2 =*/ 50).string(message.distinctId) - } - if (message.createdAt != null && Object.hasOwnProperty.call(message, 'createdAt')) { - writer.uint32(/* id 7, wireType 2 =*/ 58).string(message.createdAt) - } - if (message.elementsChain != null && Object.hasOwnProperty.call(message, 'elementsChain')) { - writer.uint32(/* id 8, wireType 2 =*/ 66).string(message.elementsChain) - } - if (message.protoCreatedAt != null && Object.hasOwnProperty.call(message, 'protoCreatedAt')) { - $root.google.protobuf.Timestamp.encode( - message.protoCreatedAt, - writer.uint32(/* id 9, wireType 2 =*/ 74).fork() - ).ldelim() - } - if (message.protoTimestamp != null && Object.hasOwnProperty.call(message, 'protoTimestamp')) { - $root.google.protobuf.Timestamp.encode( - message.protoTimestamp, - writer.uint32(/* id 10, wireType 2 =*/ 82).fork() - ).ldelim() - } - return writer - } - - /** - * Encodes the specified Event message, length delimited. Does not implicitly {@link Event.verify|verify} messages. - * @function encodeDelimited - * @memberof Event - * @static - * @param {IEvent} message Event message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Event.encodeDelimited = function encodeDelimited(message, writer) { - return this.encode(message, writer).ldelim() - } - - /** - * Decodes an Event message from the specified reader or buffer. - * @function decode - * @memberof Event - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @param {number} [length] Message length if known beforehand - * @returns {Event} Event - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Event.decode = function decode(reader, length) { - if (!(reader instanceof $Reader)) { - reader = $Reader.create(reader) - } - var end = length === undefined ? reader.len : reader.pos + length, - message = new $root.Event() - while (reader.pos < end) { - var tag = reader.uint32() - switch (tag >>> 3) { - case 1: - message.uuid = reader.string() - break - case 2: - message.event = reader.string() - break - case 3: - message.properties = reader.string() - break - case 4: - message.timestamp = reader.string() - break - case 5: - message.teamId = reader.uint64() - break - case 6: - message.distinctId = reader.string() - break - case 7: - message.createdAt = reader.string() - break - case 8: - message.elementsChain = reader.string() - break - case 9: - message.protoCreatedAt = $root.google.protobuf.Timestamp.decode(reader, reader.uint32()) - break - case 10: - message.protoTimestamp = $root.google.protobuf.Timestamp.decode(reader, reader.uint32()) - break - default: - reader.skipType(tag & 7) - break - } - } - return message - } - - /** - * Decodes an Event message from the specified reader or buffer, length delimited. - * @function decodeDelimited - * @memberof Event - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @returns {Event} Event - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Event.decodeDelimited = function decodeDelimited(reader) { - if (!(reader instanceof $Reader)) { - reader = new $Reader(reader) - } - return this.decode(reader, reader.uint32()) - } - - /** - * Verifies an Event message. - * @function verify - * @memberof Event - * @static - * @param {Object.} message Plain object to verify - * @returns {string|null} `null` if valid, otherwise the reason why it is not - */ - Event.verify = function verify(message) { - if (typeof message !== 'object' || message === null) { - return 'object expected' - } - if (message.uuid != null && message.hasOwnProperty('uuid')) { - if (!$util.isString(message.uuid)) { - return 'uuid: string expected' - } - } - if (message.event != null && message.hasOwnProperty('event')) { - if (!$util.isString(message.event)) { - return 'event: string expected' - } - } - if (message.properties != null && message.hasOwnProperty('properties')) { - if (!$util.isString(message.properties)) { - return 'properties: string expected' - } - } - if (message.timestamp != null && message.hasOwnProperty('timestamp')) { - if (!$util.isString(message.timestamp)) { - return 'timestamp: string expected' - } - } - if (message.teamId != null && message.hasOwnProperty('teamId')) { - if ( - !$util.isInteger(message.teamId) && - !(message.teamId && $util.isInteger(message.teamId.low) && $util.isInteger(message.teamId.high)) - ) { - return 'teamId: integer|Long expected' - } - } - if (message.distinctId != null && message.hasOwnProperty('distinctId')) { - if (!$util.isString(message.distinctId)) { - return 'distinctId: string expected' - } - } - if (message.createdAt != null && message.hasOwnProperty('createdAt')) { - if (!$util.isString(message.createdAt)) { - return 'createdAt: string expected' - } - } - if (message.elementsChain != null && message.hasOwnProperty('elementsChain')) { - if (!$util.isString(message.elementsChain)) { - return 'elementsChain: string expected' - } - } - if (message.protoCreatedAt != null && message.hasOwnProperty('protoCreatedAt')) { - var error = $root.google.protobuf.Timestamp.verify(message.protoCreatedAt) - if (error) { - return 'protoCreatedAt.' + error - } - } - if (message.protoTimestamp != null && message.hasOwnProperty('protoTimestamp')) { - var error = $root.google.protobuf.Timestamp.verify(message.protoTimestamp) - if (error) { - return 'protoTimestamp.' + error - } - } - return null - } - - /** - * Creates an Event message from a plain object. Also converts values to their respective internal types. - * @function fromObject - * @memberof Event - * @static - * @param {Object.} object Plain object - * @returns {Event} Event - */ - Event.fromObject = function fromObject(object) { - if (object instanceof $root.Event) { - return object - } - var message = new $root.Event() - if (object.uuid != null) { - message.uuid = String(object.uuid) - } - if (object.event != null) { - message.event = String(object.event) - } - if (object.properties != null) { - message.properties = String(object.properties) - } - if (object.timestamp != null) { - message.timestamp = String(object.timestamp) - } - if (object.teamId != null) { - if ($util.Long) { - ;(message.teamId = $util.Long.fromValue(object.teamId)).unsigned = true - } else if (typeof object.teamId === 'string') { - message.teamId = parseInt(object.teamId, 10) - } else if (typeof object.teamId === 'number') { - message.teamId = object.teamId - } else if (typeof object.teamId === 'object') { - message.teamId = new $util.LongBits(object.teamId.low >>> 0, object.teamId.high >>> 0).toNumber(true) - } - } - if (object.distinctId != null) { - message.distinctId = String(object.distinctId) - } - if (object.createdAt != null) { - message.createdAt = String(object.createdAt) - } - if (object.elementsChain != null) { - message.elementsChain = String(object.elementsChain) - } - if (object.protoCreatedAt != null) { - if (typeof object.protoCreatedAt !== 'object') { - throw TypeError('.Event.protoCreatedAt: object expected') - } - message.protoCreatedAt = $root.google.protobuf.Timestamp.fromObject(object.protoCreatedAt) - } - if (object.protoTimestamp != null) { - if (typeof object.protoTimestamp !== 'object') { - throw TypeError('.Event.protoTimestamp: object expected') - } - message.protoTimestamp = $root.google.protobuf.Timestamp.fromObject(object.protoTimestamp) - } - return message - } - - /** - * Creates a plain object from an Event message. Also converts values to other types if specified. - * @function toObject - * @memberof Event - * @static - * @param {Event} message Event - * @param {$protobuf.IConversionOptions} [options] Conversion options - * @returns {Object.} Plain object - */ - Event.toObject = function toObject(message, options) { - if (!options) { - options = {} - } - var object = {} - if (options.defaults) { - object.uuid = '' - object.event = '' - object.properties = '' - object.timestamp = '' - if ($util.Long) { - var long = new $util.Long(0, 0, true) - object.teamId = - options.longs === String ? long.toString() : options.longs === Number ? long.toNumber() : long - } else { - object.teamId = options.longs === String ? '0' : 0 - } - object.distinctId = '' - object.createdAt = '' - object.elementsChain = '' - object.protoCreatedAt = null - object.protoTimestamp = null - } - if (message.uuid != null && message.hasOwnProperty('uuid')) { - object.uuid = message.uuid - } - if (message.event != null && message.hasOwnProperty('event')) { - object.event = message.event - } - if (message.properties != null && message.hasOwnProperty('properties')) { - object.properties = message.properties - } - if (message.timestamp != null && message.hasOwnProperty('timestamp')) { - object.timestamp = message.timestamp - } - if (message.teamId != null && message.hasOwnProperty('teamId')) { - if (typeof message.teamId === 'number') { - object.teamId = options.longs === String ? String(message.teamId) : message.teamId - } else { - object.teamId = - options.longs === String - ? $util.Long.prototype.toString.call(message.teamId) - : options.longs === Number - ? new $util.LongBits(message.teamId.low >>> 0, message.teamId.high >>> 0).toNumber(true) - : message.teamId - } - } - if (message.distinctId != null && message.hasOwnProperty('distinctId')) { - object.distinctId = message.distinctId - } - if (message.createdAt != null && message.hasOwnProperty('createdAt')) { - object.createdAt = message.createdAt - } - if (message.elementsChain != null && message.hasOwnProperty('elementsChain')) { - object.elementsChain = message.elementsChain - } - if (message.protoCreatedAt != null && message.hasOwnProperty('protoCreatedAt')) { - object.protoCreatedAt = $root.google.protobuf.Timestamp.toObject(message.protoCreatedAt, options) - } - if (message.protoTimestamp != null && message.hasOwnProperty('protoTimestamp')) { - object.protoTimestamp = $root.google.protobuf.Timestamp.toObject(message.protoTimestamp, options) - } - return object - } - - /** - * Converts this Event to JSON. - * @function toJSON - * @memberof Event - * @instance - * @returns {Object.} JSON object - */ - Event.prototype.toJSON = function toJSON() { - return this.constructor.toObject(this, $protobuf.util.toJSONOptions) - } - - return Event -})() - -$root.google = (function () { - /** - * Namespace google. - * @exports google - * @namespace - */ - var google = {} - - google.protobuf = (function () { - /** - * Namespace protobuf. - * @memberof google - * @namespace - */ - var protobuf = {} - - protobuf.Timestamp = (function () { - /** - * Properties of a Timestamp. - * @memberof google.protobuf - * @interface ITimestamp - * @property {number|Long|null} [seconds] Timestamp seconds - * @property {number|null} [nanos] Timestamp nanos - */ - - /** - * Constructs a new Timestamp. - * @memberof google.protobuf - * @classdesc Represents a Timestamp. - * @implements ITimestamp - * @constructor - * @param {google.protobuf.ITimestamp=} [properties] Properties to set - */ - function Timestamp(properties) { - if (properties) { - for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) { - if (properties[keys[i]] != null) { - this[keys[i]] = properties[keys[i]] - } - } - } - } - - /** - * Timestamp seconds. - * @member {number|Long} seconds - * @memberof google.protobuf.Timestamp - * @instance - */ - Timestamp.prototype.seconds = $util.Long ? $util.Long.fromBits(0, 0, false) : 0 - - /** - * Timestamp nanos. - * @member {number} nanos - * @memberof google.protobuf.Timestamp - * @instance - */ - Timestamp.prototype.nanos = 0 - - /** - * Creates a new Timestamp instance using the specified properties. - * @function create - * @memberof google.protobuf.Timestamp - * @static - * @param {google.protobuf.ITimestamp=} [properties] Properties to set - * @returns {google.protobuf.Timestamp} Timestamp instance - */ - Timestamp.create = function create(properties) { - return new Timestamp(properties) - } - - /** - * Encodes the specified Timestamp message. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. - * @function encode - * @memberof google.protobuf.Timestamp - * @static - * @param {google.protobuf.ITimestamp} message Timestamp message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Timestamp.encode = function encode(message, writer) { - if (!writer) { - writer = $Writer.create() - } - if (message.seconds != null && Object.hasOwnProperty.call(message, 'seconds')) { - writer.uint32(/* id 1, wireType 0 =*/ 8).int64(message.seconds) - } - if (message.nanos != null && Object.hasOwnProperty.call(message, 'nanos')) { - writer.uint32(/* id 2, wireType 0 =*/ 16).int32(message.nanos) - } - return writer - } - - /** - * Encodes the specified Timestamp message, length delimited. Does not implicitly {@link google.protobuf.Timestamp.verify|verify} messages. - * @function encodeDelimited - * @memberof google.protobuf.Timestamp - * @static - * @param {google.protobuf.ITimestamp} message Timestamp message or plain object to encode - * @param {$protobuf.Writer} [writer] Writer to encode to - * @returns {$protobuf.Writer} Writer - */ - Timestamp.encodeDelimited = function encodeDelimited(message, writer) { - return this.encode(message, writer).ldelim() - } - - /** - * Decodes a Timestamp message from the specified reader or buffer. - * @function decode - * @memberof google.protobuf.Timestamp - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @param {number} [length] Message length if known beforehand - * @returns {google.protobuf.Timestamp} Timestamp - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Timestamp.decode = function decode(reader, length) { - if (!(reader instanceof $Reader)) { - reader = $Reader.create(reader) - } - var end = length === undefined ? reader.len : reader.pos + length, - message = new $root.google.protobuf.Timestamp() - while (reader.pos < end) { - var tag = reader.uint32() - switch (tag >>> 3) { - case 1: - message.seconds = reader.int64() - break - case 2: - message.nanos = reader.int32() - break - default: - reader.skipType(tag & 7) - break - } - } - return message - } - - /** - * Decodes a Timestamp message from the specified reader or buffer, length delimited. - * @function decodeDelimited - * @memberof google.protobuf.Timestamp - * @static - * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from - * @returns {google.protobuf.Timestamp} Timestamp - * @throws {Error} If the payload is not a reader or valid buffer - * @throws {$protobuf.util.ProtocolError} If required fields are missing - */ - Timestamp.decodeDelimited = function decodeDelimited(reader) { - if (!(reader instanceof $Reader)) { - reader = new $Reader(reader) - } - return this.decode(reader, reader.uint32()) - } - - /** - * Verifies a Timestamp message. - * @function verify - * @memberof google.protobuf.Timestamp - * @static - * @param {Object.} message Plain object to verify - * @returns {string|null} `null` if valid, otherwise the reason why it is not - */ - Timestamp.verify = function verify(message) { - if (typeof message !== 'object' || message === null) { - return 'object expected' - } - if (message.seconds != null && message.hasOwnProperty('seconds')) { - if ( - !$util.isInteger(message.seconds) && - !( - message.seconds && - $util.isInteger(message.seconds.low) && - $util.isInteger(message.seconds.high) - ) - ) { - return 'seconds: integer|Long expected' - } - } - if (message.nanos != null && message.hasOwnProperty('nanos')) { - if (!$util.isInteger(message.nanos)) { - return 'nanos: integer expected' - } - } - return null - } - - /** - * Creates a Timestamp message from a plain object. Also converts values to their respective internal types. - * @function fromObject - * @memberof google.protobuf.Timestamp - * @static - * @param {Object.} object Plain object - * @returns {google.protobuf.Timestamp} Timestamp - */ - Timestamp.fromObject = function fromObject(object) { - if (object instanceof $root.google.protobuf.Timestamp) { - return object - } - var message = new $root.google.protobuf.Timestamp() - if (object.seconds != null) { - if ($util.Long) { - ;(message.seconds = $util.Long.fromValue(object.seconds)).unsigned = false - } else if (typeof object.seconds === 'string') { - message.seconds = parseInt(object.seconds, 10) - } else if (typeof object.seconds === 'number') { - message.seconds = object.seconds - } else if (typeof object.seconds === 'object') { - message.seconds = new $util.LongBits( - object.seconds.low >>> 0, - object.seconds.high >>> 0 - ).toNumber() - } - } - if (object.nanos != null) { - message.nanos = object.nanos | 0 - } - return message - } - - /** - * Creates a plain object from a Timestamp message. Also converts values to other types if specified. - * @function toObject - * @memberof google.protobuf.Timestamp - * @static - * @param {google.protobuf.Timestamp} message Timestamp - * @param {$protobuf.IConversionOptions} [options] Conversion options - * @returns {Object.} Plain object - */ - Timestamp.toObject = function toObject(message, options) { - if (!options) { - options = {} - } - var object = {} - if (options.defaults) { - if ($util.Long) { - var long = new $util.Long(0, 0, false) - object.seconds = - options.longs === String - ? long.toString() - : options.longs === Number - ? long.toNumber() - : long - } else { - object.seconds = options.longs === String ? '0' : 0 - } - object.nanos = 0 - } - if (message.seconds != null && message.hasOwnProperty('seconds')) { - if (typeof message.seconds === 'number') { - object.seconds = options.longs === String ? String(message.seconds) : message.seconds - } else { - object.seconds = - options.longs === String - ? $util.Long.prototype.toString.call(message.seconds) - : options.longs === Number - ? new $util.LongBits(message.seconds.low >>> 0, message.seconds.high >>> 0).toNumber() - : message.seconds - } - } - if (message.nanos != null && message.hasOwnProperty('nanos')) { - object.nanos = message.nanos - } - return object - } - - /** - * Converts this Timestamp to JSON. - * @function toJSON - * @memberof google.protobuf.Timestamp - * @instance - * @returns {Object.} JSON object - */ - Timestamp.prototype.toJSON = function toJSON() { - return this.constructor.toObject(this, $protobuf.util.toJSONOptions) - } - - return Timestamp - })() - - return protobuf - })() - - return google -})() - -module.exports = $root From 70c9e43e496e2fcbedec0315438fee66242b37a5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 13 Jan 2021 16:29:50 +0100 Subject: [PATCH 087/218] Rename consumer group to clickhouse-ingestion --- src/ingestion/kafka-queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 8c26a598..9edfe779 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -113,7 +113,7 @@ export class KafkaQueue implements Queue { private static buildConsumer(kafka: Kafka): Consumer { const consumer = kafka.consumer({ - groupId: 'group1', + groupId: 'clickhouse-ingestion', readUncommitted: false, }) const { GROUP_JOIN, CRASH, CONNECT, DISCONNECT } = consumer.events From f326b804637943662d63a6921c64491cf49302f0 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 15 Jan 2021 09:15:56 +0100 Subject: [PATCH 088/218] Consume from topic events_ingestion_handoff --- src/extensions/posthog.ts | 4 ++-- src/ingestion/kafka-queue.ts | 10 +++++----- src/ingestion/topics.ts | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index e6bcd5c9..7e19643b 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -1,4 +1,4 @@ -import { KAFKA_EVENTS_WAL } from '../ingestion/topics' +import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../ingestion/topics' import { DateTime } from 'luxon' import { PluginsServer, PluginConfig, RawEventMessage } from 'types' import { version } from '../../package.json' @@ -51,7 +51,7 @@ export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig) } producer!.send({ - topic: KAFKA_EVENTS_WAL, + topic: KAFKA_EVENTS_INGESTION_HANDOFF, messages: [ { key: uuid, diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 9edfe779..4015875c 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -1,7 +1,7 @@ import * as Sentry from '@sentry/node' import { Kafka, Consumer, Message } from 'kafkajs' import { ParsedEventMessage, PluginsServer, Queue, RawEventMessage } from 'types' -import { KAFKA_EVENTS_WAL } from './topics' +import { KAFKA_EVENTS_INGESTION_HANDOFF } from './topics' import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' @@ -30,7 +30,7 @@ export class KafkaQueue implements Queue { async start(): Promise { status.info('โฌ', `Connecting Kafka consumer to ${this.pluginsServer.KAFKA_HOSTS}...`) - await this.consumer.subscribe({ topic: KAFKA_EVENTS_WAL }) + await this.consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) // KafkaJS batching: https://kafka.js.org/docs/consuming#a-name-each-batch-a-eachbatch await this.consumer.run({ // TODO: eachBatchAutoResolve: false, // don't autoresolve whole batch in case we exit it early @@ -87,7 +87,7 @@ export class KafkaQueue implements Queue { return } console.error('โณ Pausing Kafka consumer...') - await this.consumer.pause([{ topic: KAFKA_EVENTS_WAL }]) + await this.consumer.pause([{ topic: KAFKA_EVENTS_INGESTION_HANDOFF }]) console.error('โธ Kafka consumer paused!') } @@ -96,12 +96,12 @@ export class KafkaQueue implements Queue { return } console.error('โณ Resuming Kafka consumer...') - await this.consumer.resume([{ topic: KAFKA_EVENTS_WAL }]) + await this.consumer.resume([{ topic: KAFKA_EVENTS_INGESTION_HANDOFF }]) console.error('โ–ถ๏ธ Kafka consumer resumed!') } isPaused(): boolean { - return this.consumer.paused().some(({ topic }) => topic === KAFKA_EVENTS_WAL) + return this.consumer.paused().some(({ topic }) => topic === KAFKA_EVENTS_INGESTION_HANDOFF) } async stop(): Promise { diff --git a/src/ingestion/topics.ts b/src/ingestion/topics.ts index c90ebf55..05c6de36 100644 --- a/src/ingestion/topics.ts +++ b/src/ingestion/topics.ts @@ -3,3 +3,4 @@ export const KAFKA_PERSON = 'clickhouse_person' export const KAFKA_PERSON_UNIQUE_ID = 'clickhouse_person_unique_id' export const KAFKA_SESSION_RECORDING_EVENTS = 'clickhouse_session_recording_events' export const KAFKA_EVENTS_WAL = 'events_write_ahead_log' +export const KAFKA_EVENTS_INGESTION_HANDOFF = 'events_ingestion_handoff' From 5f1c17d803f1437e772f9d009429dcce0d25a1c8 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 15 Jan 2021 14:42:45 +0100 Subject: [PATCH 089/218] ee dev script --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 4803910e..46a04c3b 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", "start:dev": "ts-node-dev --exit-child src/index.ts --base-dir ../posthog", + "start:dev:ee": "DATABASE_URL=postgres://posthog:posthog@localhost:5439/posthog KAFKA_ENABLED=true KAFKA_HOSTS=localhost:9092 yarn start:dev", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && eslint --fix . && prettier --write .", From 737a8c77302931eadb3fbe42b45d9e4420782e90 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 19 Jan 2021 10:31:04 +0100 Subject: [PATCH 090/218] Backport set_once and event name sanitization --- src/ingestion/process-event.ts | 40 +++++++++++++++++--------- src/ingestion/{element.ts => utils.ts} | 13 ++++++++- 2 files changed, 38 insertions(+), 15 deletions(-) rename src/ingestion/{element.ts => utils.ts} (80%) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 0ff67221..b23cfaf3 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,12 +1,12 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { DateTime, Duration } from 'luxon' -import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from 'types' +import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from '../types' import { castTimestampOrNow, UUID, UUIDT } from '../utils' -import { elementsToString } from './element' import { Event as EventProto } from '../idl/protos' import { Pool } from 'pg' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' +import { sanitizeEventName, elementsToString } from './utils' export class EventsProcessor { pluginsServer: PluginsServer @@ -33,6 +33,9 @@ export class EventsProcessor { if (data['$set']) { properties['$set'] = data['$set'] } + if (data['$set_once']) { + properties['$set_once'] = data['$set_once'] + } const personUuid = new UUIDT() @@ -97,9 +100,13 @@ export class EventsProcessor { if (properties['$anon_distinct_id']) { await this.alias(properties['$anon_distinct_id'], distinctId, teamId) } + // TODO: roll the two updatePersonProperties calls into one to `UPDATE posthog_person` only once if (properties['$set']) { this.updatePersonProperties(teamId, distinctId, properties['$set']) } + if (properties['$set_once']) { + this.updatePersonProperties(teamId, distinctId, properties['$set_once'], true) + } this.setIsIdentified(teamId, distinctId) } } @@ -124,7 +131,12 @@ export class EventsProcessor { } } - private async updatePersonProperties(teamId: number, distinctId: string, properties: Properties): Promise { + private async updatePersonProperties( + teamId: number, + distinctId: string, + properties: Properties, + setOnce = false + ): Promise { let personFound = await this.fetchPerson(teamId, distinctId) if (!personFound) { try { @@ -143,12 +155,10 @@ export class EventsProcessor { personFound = await this.fetchPerson(teamId, distinctId) } } - if (personFound) { - this.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ - { ...personFound.properties, ...properties }, - personFound.id, - ]) - } + this.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ + setOnce ? { ...properties, ...personFound!.properties } : { ...personFound!.properties, ...properties }, + personFound!.id, + ]) } private async alias( @@ -267,6 +277,8 @@ export class EventsProcessor { timestamp: DateTime, sentAt: DateTime | null ): Promise { + event = sanitizeEventName(event) + const elements: Record[] | undefined = properties['$elements'] let elements_list: Element[] = [] if (elements && elements.length) { @@ -352,11 +364,11 @@ export class EventsProcessor { WHERE id = $7`, [ team.ingested_event, - JSON.stringify(team.event_names), - JSON.stringify(team.event_names_with_usage), - JSON.stringify(team.event_properties), - JSON.stringify(team.event_names_with_usage), - JSON.stringify(team.event_properties_numerical), + team.event_names, + team.event_names_with_usage, + team.event_properties, + team.event_names_with_usage, + team.event_properties_numerical, team.id, ] ) diff --git a/src/ingestion/element.ts b/src/ingestion/utils.ts similarity index 80% rename from src/ingestion/element.ts rename to src/ingestion/utils.ts index d17deb1f..398a3f92 100644 --- a/src/ingestion/element.ts +++ b/src/ingestion/utils.ts @@ -1,6 +1,6 @@ import { Element } from 'types' -function escapeQuotes(input: string): string { +export function escapeQuotes(input: string): string { return input.replace(/"/g, '\\"') } @@ -37,3 +37,14 @@ export function elementsToString(elements: Element[]): string { }) return ret.join(';') } + +export function sanitizeEventName(eventName: any): string { + if (typeof eventName !== 'string') { + try { + eventName = JSON.stringify(eventName) + } catch { + eventName = String(eventName) + } + } + return eventName.substr(0, 2) +} From bacf5744799db4918de91f8ac3c47dd14c30c0c6 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 19 Jan 2021 12:19:31 +0100 Subject: [PATCH 091/218] Add direct ClickHouse support base --- package.json | 1 + src/config.ts | 14 +++++ src/ingestion/process-event.ts | 3 +- src/server.ts | 13 +++++ src/types.ts | 7 +++ yarn.lock | 94 ++++++++++++++++++++++++++++++++-- 6 files changed, 127 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 46a04c3b..1b5f4596 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@sentry/node": "^5.29.0", "@sentry/tracing": "^5.29.0", "adm-zip": "^0.4.16", + "clickhouse": "^2.2.1", "fastify": "^3.8.0", "hot-shots": "^8.2.1", "ioredis": "^4.19.2", diff --git a/src/config.ts b/src/config.ts index 7527a10b..8efbd397 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,6 +9,13 @@ export function getDefaultConfig(): PluginsServerConfig { return { CELERY_DEFAULT_QUEUE: 'celery', DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/posthog_test' : 'postgres://localhost:5432/posthog', + CLICKHOUSE_HOST: 'localhost', + CLICKHOUSE_DATABASE: process.env.NODE_ENV === 'test' ? 'posthog_test' : 'default', + CLICKHOUSE_USERNAME: 'default', + CLICKHOUSE_PASSWORD: null, + CLICKHOUSE_CA: null, + CLICKHOUSE_SECURE: false, + CLICKHOUSE_VERIFY: true, KAFKA_ENABLED: false, KAFKA_HOSTS: null, KAFKA_CLIENT_CERT_B64: null, @@ -37,6 +44,13 @@ export function getConfigHelp(): Record { CELERY_DEFAULT_QUEUE: 'Celery outgoing queue', PLUGINS_CELERY_QUEUE: 'Celery incoming queue', DATABASE_URL: 'Postgres database URL', + CLICKHOUSE_HOST: 'ClickHouse host', + CLICKHOUSE_DATABASE: 'ClickHouse database', + CLICKHOUSE_USERNAME: 'ClickHouse username', + CLICKHOUSE_PASSWORD: 'ClickHouse password', + CLICKHOUSE_CA: 'ClickHouse CA certs', + CLICKHOUSE_SECURE: 'Secure ClickHouse connection', + CLICKHOUSE_VERIFY: 'Verify ClickHouse connection', REDIS_URL: 'Redis store URL', BASE_DIR: 'base path for resolving local plugins', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'Redis channel for reload events', diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index b23cfaf3..e56e2c45 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -405,7 +405,8 @@ export class EventsProcessor { 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', [createdAt.toISO(), properties, teamId, isUserId, isIdentified, uuid.toString()] ) - return insertResult.rows[0] + const personCreated = insertResult.rows[0] as Person + return personCreated } private async addDistinctId(person: Person, distinctId: string): Promise { diff --git a/src/server.ts b/src/server.ts index cc5eeac1..bda94108 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,7 @@ import { Kafka, logLevel, Producer } from 'kafkajs' import { FastifyInstance } from 'fastify' import { PluginsServer, PluginsServerConfig, Queue } from './types' import { startQueue } from './worker/queue' +import { ClickHouse } from 'clickhouse' import { startFastifyInstance, stopFastifyInstance } from './web/server' import { version } from '../package.json' import { PluginEvent } from '@posthog/plugin-scaffold' @@ -64,12 +65,24 @@ export async function createServer( } } + let clickhouse: ClickHouse | undefined let kafka: Kafka | undefined let kafkaProducer: Producer | undefined if (serverConfig.KAFKA_ENABLED) { if (!serverConfig.KAFKA_HOSTS) { throw new Error('You must set KAFKA_HOSTS to process events from Kafka!') } + clickhouse = new ClickHouse({ + url: `http${serverConfig.CLICKHOUSE_SECURE ? 's' : ''}://$${serverConfig.CLICKHOUSE_HOST}`, + port: serverConfig.CLICKHOUSE_SECURE ? 8443 : 8123, + basicAuth: { + username: serverConfig.CLICKHOUSE_USERNAME, + password: serverConfig.CLICKHOUSE_PASSWORD, + }, + config: { + database: serverConfig.CLICKHOUSE_DATABASE, + }, + }) kafka = new Kafka({ clientId: `plugin-server-v${version}`, brokers: serverConfig.KAFKA_HOSTS.split(','), diff --git a/src/types.ts b/src/types.ts index 71cc2268..3901ab10 100644 --- a/src/types.ts +++ b/src/types.ts @@ -22,6 +22,13 @@ export interface PluginsServerConfig extends Record { TASKS_PER_WORKER: number CELERY_DEFAULT_QUEUE: string DATABASE_URL: string + CLICKHOUSE_HOST: string + CLICKHOUSE_DATABASE: string + CLICKHOUSE_USERNAME: string + CLICKHOUSE_PASSWORD: string | null + CLICKHOUSE_CA: string | null + CLICKHOUSE_SECURE: boolean + CLICKHOUSE_VERIFY: boolean KAFKA_ENABLED: boolean KAFKA_HOSTS: string | null KAFKA_CLIENT_CERT_B64: string | null diff --git a/yarn.lock b/yarn.lock index 1535d3b2..f07097b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1619,6 +1619,14 @@ "@typescript-eslint/types" "4.9.1" eslint-visitor-keys "^2.0.0" +JSONStream@1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.4.tgz#615bb2adb0cd34c8f4c447b5f6512fa1d8f16a2e" + integrity sha512-Y7vfi3I5oMOYIr+WxV8NZxDSwcbNgzdKYsTNInmycOq9bUYwGg9ryu57Wg5NLmCjqdFPNUmpMBo3kSJN9tCbXg== + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + abab@^2.0.3: version "2.0.5" resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" @@ -2280,6 +2288,20 @@ cli-truncate@^2.1.0: slice-ansi "^3.0.0" string-width "^4.2.0" +clickhouse@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/clickhouse/-/clickhouse-2.2.1.tgz#dc7c6c24a54a829ed2e7c4d5495852afe15ded1e" + integrity sha512-NcqKA1Uc353QKs1MepTSZZrv3gF4v5ScoobAoDxcC+37VKV1QusG/hkaZHTupJ28oppuRFpdWKSIoN54qkD2vg== + dependencies: + JSONStream "1.3.4" + lodash "4.17.19" + querystring "0.2.0" + request "2.88.0" + stream2asynciter "1.0.1" + through "2.3.8" + tsv "0.2.0" + uuid "3.4.0" + cliui@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-6.0.0.tgz#511d702c0c4e41ca156d7d0e96021f23e13225b1" @@ -3677,7 +3699,7 @@ har-schema@^2.0.0: resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= -har-validator@~5.1.3: +har-validator@~5.1.0, har-validator@~5.1.3: version "5.1.5" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== @@ -4739,6 +4761,11 @@ json5@^1.0.1: dependencies: minimist "^1.2.0" +jsonparse@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= + jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -4936,6 +4963,11 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg= +lodash@4.17.19: + version "4.17.19" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b" + integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ== + lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20: version "4.17.20" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" @@ -5882,7 +5914,7 @@ proxy-addr@^2.0.5: forwarded "~0.1.2" ipaddr.js "1.9.1" -psl@^1.1.28: +psl@^1.1.24, psl@^1.1.28: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== @@ -5895,6 +5927,11 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" +punycode@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= + punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" @@ -5910,6 +5947,11 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + queue-microtask@^1.1.2: version "1.2.2" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.2.tgz#abf64491e6ecf0f38a6502403d4cda04f372dfd3" @@ -6165,6 +6207,32 @@ request-promise-native@^1.0.8: stealthy-require "^1.1.1" tough-cookie "^2.3.3" +request@2.88.0: + version "2.88.0" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" + integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.0" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.4.3" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + request@^2.88.2: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" @@ -6678,6 +6746,11 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +stream2asynciter@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/stream2asynciter/-/stream2asynciter-1.0.1.tgz#9f35c8fff36d89c02030b72e36702b61b6224d10" + integrity sha512-HseDrFlR3YhQEnZ4MAoQmW8poOkUeVfRiuv6t3+Wwb2ehNs4SSgzENcZoYEQdCgp/OFkt1IL2DpyvG9VctQXjg== + string-argv@0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da" @@ -6940,7 +7013,7 @@ throat@^5.0.0: resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b" integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA== -through@^2.3.8: +through@2.3.8, "through@>=2.2.7 <3", through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= @@ -7016,6 +7089,14 @@ tough-cookie@^3.0.1: psl "^1.1.28" punycode "^2.1.1" +tough-cookie@~2.4.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" + integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ== + dependencies: + psl "^1.1.24" + punycode "^1.4.1" + tr46@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.0.2.tgz#03273586def1595ae08fedb38d7733cee91d2479" @@ -7111,6 +7192,11 @@ tsutils@^3.17.1: dependencies: tslib "^1.8.1" +tsv@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/tsv/-/tsv-0.2.0.tgz#92869a3cb5f50332f3dc90fca82be667db6f72d6" + integrity sha1-koaaPLX1AzLz3JD8qCvmZ9tvctY= + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" @@ -7255,7 +7341,7 @@ util-deprecate@^1.0.1, util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= -uuid@^3.3.2: +uuid@3.4.0, uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== From 757dd711e687cd05cbbcb39c81a9180ed263cc82 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 19 Jan 2021 14:50:23 +0100 Subject: [PATCH 092/218] Upgrade plugin-scaffold to 0.2.8 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 1b5f4596..219f30e1 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "@babel/core": "^7.0.0", "@babel/preset-env": "^7.0.0", "@babel/preset-typescript": "^7.8.3", - "@posthog/plugin-scaffold": "0.2.7", + "@posthog/plugin-scaffold": "0.2.8", "@types/adm-zip": "^0.4.33", "@types/ioredis": "^4.17.7", "@types/jest": "^26.0.15", diff --git a/yarn.lock b/yarn.lock index f07097b7..f4dce87f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1179,10 +1179,10 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" -"@posthog/plugin-scaffold@0.2.7": - version "0.2.7" - resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.7.tgz#0d4a87385ec5c88184ad252a2587b9793f50a58d" - integrity sha512-bgrXLwUMMJ5TzyNK41jlgKdpZffvwjDxYeZ7nL1VDAI/Svv4/DSIA7JtcXZV14q2x99vB3woPckWUUAC7K2jmg== +"@posthog/plugin-scaffold@0.2.8": + version "0.2.8" + resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.8.tgz#14ee85afa9a91625bbae4e223ce03d215a22e8d7" + integrity sha512-RcjmiPe3fAgLR/qyqTxnTg3b9WdoOAeV4tEPvisEFnej3mLenxp7bgUY5W1lxSfYoDDLGx1zSRVnBQL56w5KTQ== "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" From 732fb64862c6992f37567db5b778105af5877aaa Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 19 Jan 2021 14:50:30 +0100 Subject: [PATCH 093/218] Add clickhouse to server object --- src/server.ts | 1 + src/types.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/server.ts b/src/server.ts index bda94108..e6bcb9f9 100644 --- a/src/server.ts +++ b/src/server.ts @@ -112,6 +112,7 @@ export async function createServer( ...serverConfig, db, redis, + clickhouse, kafka, kafkaProducer, statsd, diff --git a/src/types.ts b/src/types.ts index 3901ab10..1a601602 100644 --- a/src/types.ts +++ b/src/types.ts @@ -7,6 +7,7 @@ import { DateTime } from 'luxon' import { StatsD } from 'hot-shots' import { EventsProcessor } from 'ingestion/process-event' import { UUID } from './utils' +import { ClickHouse } from 'clickhouse' export enum LogLevel { Debug = 'debug', @@ -50,12 +51,13 @@ export interface PluginsServerConfig extends Record { } export interface PluginsServer extends PluginsServerConfig { - // active connections to Postgres, Redis, Kafka, StatsD + // active connections to Postgres, Redis, ClickHouse, Kafka, StatsD db: Pool redis: Redis - kafka: Kafka | undefined - kafkaProducer: Producer | undefined - statsd: StatsD | undefined + clickhouse?: ClickHouse + kafka?: Kafka + kafkaProducer?: Producer + statsd?: StatsD // currently enabled plugin status plugins: Map pluginConfigs: Map From 915bdbfa978d1d35b95f405b60b4f87c58c9998d Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 19 Jan 2021 17:32:06 +0100 Subject: [PATCH 094/218] Support ClickHouse persons and person distinct IDs --- src/ingestion/kafka-queue.ts | 20 ++-- src/ingestion/process-event.ts | 171 ++++++++++++++++++++++++++------- src/ingestion/utils.ts | 20 +++- src/types.ts | 50 ++++++---- src/utils.ts | 2 +- src/worker/queue.ts | 2 +- 6 files changed, 197 insertions(+), 68 deletions(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 4015875c..a1459a50 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -1,9 +1,10 @@ import * as Sentry from '@sentry/node' import { Kafka, Consumer, Message } from 'kafkajs' -import { ParsedEventMessage, PluginsServer, Queue, RawEventMessage } from 'types' +import { RawEventMessage, EventMessage, PluginsServer, Queue } from 'types' import { KAFKA_EVENTS_INGESTION_HANDOFF } from './topics' import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' +import { parseRawEventMessage } from './utils' export type BatchCallback = (messages: Message[]) => Promise @@ -51,12 +52,11 @@ export class KafkaQueue implements Queue { ...JSON.parse(message.value!.toString()), kafka_offset: message.offset, })) - const parsedEvents = rawEvents.map((rawEvent) => ({ - ...rawEvent, - data: JSON.parse(rawEvent.data), - })) + const parsedEvents = rawEvents.map(parseRawEventMessage) const pluginEvents: PluginEvent[] = parsedEvents.map((parsedEvent) => ({ ...parsedEvent, + now: parsedEvent.now.toISO(), + sent_at: parsedEvent.now.toISO(), event: parsedEvent.data.event, properties: parsedEvent.data.properties, })) @@ -105,10 +105,12 @@ export class KafkaQueue implements Queue { } async stop(): Promise { - status.info('โณ', 'Stopping Kafka queue...') - await this.consumer.stop() - status.error('โน', 'Kafka consumer stopped!') - await this.consumer.disconnect() + try { + status.info('โณ', 'Stopping Kafka queue...') + await this.consumer.stop() + status.error('โน', 'Kafka consumer stopped!') + await this.consumer.disconnect() + } catch {} } private static buildConsumer(kafka: Kafka): Consumer { diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index e56e2c45..756666c6 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,21 +1,34 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { DateTime, Duration } from 'luxon' -import { PluginsServer, EventData, Properties, Element, Team, Person, PersonDistinctId, CohortPeople } from '../types' -import { castTimestampOrNow, UUID, UUIDT } from '../utils' +import { + PluginsServer, + EventData, + Properties, + Element, + Team, + Person, + PersonDistinctId, + CohortPeople, + RawPerson, +} from '../types' +import { castTimestampOrNow, UUIDT } from '../utils' import { Event as EventProto } from '../idl/protos' import { Pool } from 'pg' import { Producer } from 'kafkajs' -import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' -import { sanitizeEventName, elementsToString } from './utils' +import { KAFKA_EVENTS, KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID, KAFKA_SESSION_RECORDING_EVENTS } from './topics' +import { sanitizeEventName, elementsToString, unparsePersonPartial } from './utils' +import { ClickHouse } from 'clickhouse' export class EventsProcessor { pluginsServer: PluginsServer db: Pool + clickhouse: ClickHouse kafkaProducer: Producer constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer this.db = pluginsServer.db + this.clickhouse = pluginsServer.clickhouse! this.kafkaProducer = pluginsServer.kafkaProducer! } @@ -27,7 +40,7 @@ export class EventsProcessor { teamId: number, now: DateTime, sentAt: DateTime | null, - eventUuid: UUID + eventUuid: string ): Promise { const properties: Properties = data.properties ?? {} if (data['$set']) { @@ -37,7 +50,7 @@ export class EventsProcessor { properties['$set_once'] = data['$set_once'] } - const personUuid = new UUIDT() + const personUuid = new UUIDT().toString() const ts = this.handleTimestamp(data, now, sentAt) this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) @@ -115,7 +128,14 @@ export class EventsProcessor { let personFound = await this.fetchPerson(teamId, distinctId) if (!personFound) { try { - const personCreated = await this.createPerson(DateTime.utc(), {}, teamId, null, true, new UUIDT()) + const personCreated = await this.createPerson( + DateTime.utc(), + {}, + teamId, + null, + true, + new UUIDT().toString() + ) this.addDistinctId(personCreated, distinctId) } catch { // Catch race condition where in between getting and creating, @@ -146,7 +166,7 @@ export class EventsProcessor { teamId, null, false, - new UUIDT() + new UUIDT().toString() ) await this.addDistinctId(personCreated, distinctId) } catch { @@ -156,7 +176,9 @@ export class EventsProcessor { } } this.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ - setOnce ? { ...properties, ...personFound!.properties } : { ...personFound!.properties, ...properties }, + JSON.stringify( + setOnce ? { ...properties, ...personFound!.properties } : { ...personFound!.properties, ...properties } + ), personFound!.id, ]) } @@ -200,7 +222,14 @@ export class EventsProcessor { if (!oldPerson && !newPerson) { try { - const personCreated = await this.createPerson(DateTime.utc(), {}, teamId, null, false, new UUIDT()) + const personCreated = await this.createPerson( + DateTime.utc(), + {}, + teamId, + null, + false, + new UUIDT().toString() + ) this.addDistinctId(personCreated, distinctId) this.addDistinctId(personCreated, previousDistinctId) } catch { @@ -231,10 +260,7 @@ export class EventsProcessor { } } - await this.db.query('UPDATE posthog_person SET created_at = $1 WHERE id = $2', [ - first_seen.toISO(), - mergeInto.id, - ]) + await this.updatePerson(mergeInto, { created_at: first_seen }) // merge the distinct_ids for (const other_person of peopleToMerge) { @@ -261,13 +287,13 @@ export class EventsProcessor { ]) } - await this.db.query('DELETE FROM posthog_person WHERE id = $1', [other_person.id]) + await this.deletePerson(other_person.id) } } private async captureEE( - eventUuid: UUID, - personUuid: UUID, + eventUuid: string, + personUuid: string, ip: string, siteUrl: string, teamId: number, @@ -364,11 +390,11 @@ export class EventsProcessor { WHERE id = $7`, [ team.ingested_event, - team.event_names, - team.event_names_with_usage, - team.event_properties, - team.event_names_with_usage, - team.event_properties_numerical, + JSON.stringify(team.event_names), + JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties), + JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties_numerical), team.id, ] ) @@ -390,7 +416,8 @@ export class EventsProcessor { AND posthog_persondistinctid.distinct_id = $2`, [teamId, distinctId] ) - return selectResult.rows[0] + const rawPerson: RawPerson = selectResult.rows[0] + return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } } private async createPerson( @@ -399,25 +426,99 @@ export class EventsProcessor { teamId: number, isUserId: number | null, isIdentified: boolean, - uuid: UUID | string + uuid: string ): Promise { const insertResult = await this.db.query( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', - [createdAt.toISO(), properties, teamId, isUserId, isIdentified, uuid.toString()] + [createdAt.toISO(), JSON.stringify(properties), teamId, isUserId, isIdentified, uuid] ) const personCreated = insertResult.rows[0] as Person + if (this.pluginsServer.KAFKA_ENABLED) { + const data = { + created_at: castTimestampOrNow(createdAt), + properties: JSON.stringify(properties), + team_id: teamId, + is_identified: isIdentified, + id: uuid, + } + await this.kafkaProducer.send({ + topic: KAFKA_PERSON, + messages: [{ value: Buffer.from(JSON.stringify(data)) }], + }) + } return personCreated } - private async addDistinctId(person: Person, distinctId: string): Promise { + private async updatePerson(person: Person, update: Partial): Promise { + const updatedPerson: Person = { ...person, ...update } await this.db.query( + `UPDATE posthog_person SET ${Object.keys(update).map( + (field, index) => field + ' = $' + (index + 1) + )} WHERE id = $${Object.values(update).length + 1}`, + [...Object.values(unparsePersonPartial(update)), person.id] + ) + if (this.pluginsServer.KAFKA_ENABLED) { + const data = { + created_at: castTimestampOrNow(updatedPerson.created_at), + properties: JSON.stringify(updatedPerson.properties), + team_id: updatedPerson.team_id, + is_identified: updatedPerson.is_identified, + id: updatedPerson.uuid.toString(), + } + await this.kafkaProducer.send({ + topic: KAFKA_PERSON, + messages: [{ value: Buffer.from(JSON.stringify(data)) }], + }) + } + return updatedPerson + } + + private async deletePerson(personId: number): Promise { + await this.db.query('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) + await this.db.query('DELETE FROM posthog_person WHERE id = $1', [personId]) + if (this.pluginsServer.KAFKA_ENABLED) { + await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() + await this.clickhouse + .query(`ALTER TABLE person_distinct_id DELETE WHERE person_id = ${personId}`) + .toPromise() + } + } + + private async addDistinctId(person: Person, distinctId: string): Promise { + const insertResult = await this.db.query( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', [distinctId, person.id, person.team_id] ) + const personDistinctIdCreated = insertResult.rows[0] as PersonDistinctId + if (this.pluginsServer.KAFKA_ENABLED) { + await this.kafkaProducer.send({ + topic: KAFKA_PERSON_UNIQUE_ID, + messages: [{ value: Buffer.from(JSON.stringify(personDistinctIdCreated)) }], + }) + } + } + + private async updateDistinctId( + personDistinctId: PersonDistinctId, + update: Partial + ): Promise { + const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } + await this.db.query( + `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( + (field, index) => field + ' = $' + (index + 1) + )} WHERE id = $${Object.values(update).length + 1}`, + [...Object.values(update), personDistinctId.id] + ) + if (this.pluginsServer.KAFKA_ENABLED) { + await this.kafkaProducer.send({ + topic: KAFKA_PERSON_UNIQUE_ID, + messages: [{ value: Buffer.from(JSON.stringify(updatedPersonDistinctId)) }], + }) + } } private async createEvent( - event_uuid: UUID, + uuid: string, event: string, team: Team, distinctId: string, @@ -427,10 +528,9 @@ export class EventsProcessor { ): Promise { const timestampString = castTimestampOrNow(timestamp) const elementsChain = elements && elements.length ? elementsToString(elements) : '' - const eventUuidString = event_uuid.toString() const message = EventProto.create({ - uuid: eventUuidString, + uuid, event, properties: JSON.stringify(properties ?? {}), timestamp: timestampString, @@ -442,14 +542,14 @@ export class EventsProcessor { await this.kafkaProducer.send({ topic: KAFKA_EVENTS, - messages: [{ key: eventUuidString, value: EventProto.encodeDelimited(message).finish() as Buffer }], + messages: [{ key: uuid, value: EventProto.encodeDelimited(message).finish() as Buffer }], }) - return eventUuidString + return uuid } private async createSessionRecordingEvent( - eventUuid: UUID, + uuid: string, team_id: number, distinct_id: string, session_id: string, @@ -457,10 +557,9 @@ export class EventsProcessor { snapshot_data: Record ): Promise { const timestampString = castTimestampOrNow(timestamp) - const eventUuidString = eventUuid.toString() const data = { - uuid: eventUuidString, + uuid, team_id: team_id, distinct_id: distinct_id, session_id: session_id, @@ -471,9 +570,9 @@ export class EventsProcessor { await this.kafkaProducer.send({ topic: KAFKA_SESSION_RECORDING_EVENTS, - messages: [{ key: eventUuidString, value: Buffer.from(JSON.stringify(data)) }], + messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }], }) - return eventUuidString + return uuid } } diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 398a3f92..c6eb0df4 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -1,4 +1,22 @@ -import { Element } from 'types' +import { DateTime } from 'luxon' +import { Element, BaseEventMessage, RawEventMessage, EventMessage, BasePerson, RawPerson, Person } from '../types' + +export function parseRawEventMessage(message: RawEventMessage): EventMessage { + return { + ...(message as BaseEventMessage), + data: JSON.parse(message.data), + now: DateTime.fromISO(message.now), + sent_at: DateTime.fromISO(message.sent_at), + } +} + +export function parseRawPerson(rawPerson: RawPerson): Person { + return { ...(rawPerson as BasePerson), created_at: DateTime.fromISO(rawPerson.created_at) } +} + +export function unparsePersonPartial(person: Partial): Partial { + return { ...(person as BasePerson), ...(person.created_at ? { created_at: person.created_at.toISO() } : {}) } +} export function escapeQuotes(input: string): string { return input.replace(/"/g, '\\"') diff --git a/src/types.ts b/src/types.ts index 1a601602..57a2deb8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,7 +6,6 @@ import { VM } from 'vm2' import { DateTime } from 'luxon' import { StatsD } from 'hot-shots' import { EventsProcessor } from 'ingestion/process-event' -import { UUID } from './utils' import { ClickHouse } from 'clickhouse' export enum LogLevel { @@ -170,23 +169,28 @@ export interface PropertyUsage { volume: number | null } -// received via Kafka -interface EventMessage { +/** Properties shared by RawEventMessage and EventMessage. */ +export interface BaseEventMessage { distinct_id: string ip: string site_url: string team_id: number + uuid: string } -export interface RawEventMessage extends EventMessage { +/** Raw event message as received via Kafka. */ +export interface RawEventMessage extends BaseEventMessage { + /** JSON-encoded object. */ data: string + /** ISO-formatted datetime. */ now: string - sent_at: string // may be an empty string - uuid: string + /** ISO-formatted datetime. May be empty! */ + sent_at: string } -export interface ParsedEventMessage extends EventMessage { - data: EventData +/** Usable event message. */ +export interface EventMessage extends BaseEventMessage { + data: PluginEvent now: DateTime sent_at: DateTime | null } @@ -198,8 +202,10 @@ export interface EventData extends PluginEvent { offset?: number } +/** Any kind of properties field. */ export type Properties = Record +/** Usable Team model. */ export interface Team { id: number name: string @@ -219,6 +225,7 @@ export interface Team { ingested_event: boolean } +/** Usable Element model. */ export interface Element { text?: string tag_name?: string @@ -233,18 +240,27 @@ export interface Element { group_id?: number } -export type User = Record // not really typed as not needed so far, only a placeholder for Person.is_user - -export interface Person { +/** Properties shared by RawPerson and Person. */ +export interface BasePerson { id: number - created_at: DateTime team_id: number properties: Properties - is_user: User + is_user_id: number is_identified: boolean uuid: string } +/** Raw Person row from database. */ +export interface RawPerson extends BasePerson { + created_at: string +} + +/** Usable Person model. */ +export interface Person extends BasePerson { + created_at: DateTime +} + +/** Usable PersonDistinctId model. */ export interface PersonDistinctId { id: number team_id: number @@ -252,15 +268,9 @@ export interface PersonDistinctId { distinct_id: string } +/** Usable CohortPeople model. */ export interface CohortPeople { id: number cohort_id: number person_id: number } - -export interface ParsedEventMessage extends EventMessage { - data: PluginEvent - now: DateTime - sent_at: DateTime | null - uuid: UUID -} diff --git a/src/utils.ts b/src/utils.ts index 27dd3206..25aa4431 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -174,9 +174,9 @@ export class UUID { /** * Convert to string format of the form: * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX - * Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js */ toString(): string { + // Adapted from https://github.com/uuidjs/uuid/blob/master/src/stringify.js // Note: Be careful editing this code! It's been tuned for performance and works in ways you may not expect. // See https://github.com/uuidjs/uuid/pull/434 const arr = this.array diff --git a/src/worker/queue.ts b/src/worker/queue.ts index f5d53ed8..9e99e532 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -81,7 +81,7 @@ function startQueueKafka( team_id, DateTime.fromISO(now), sent_at ? DateTime.fromISO(sent_at) : null, - new UUID(uuid!) + uuid! ) server.statsd?.timing('single-ingestion', singleIngestionTimer) }) From 45aa41f79188bff16cc000dfa5520b2240cf0e2e Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 19 Jan 2021 18:01:12 +0100 Subject: [PATCH 095/218] Update README with ClickHouse settings --- README.md | 6 ++++++ src/config.ts | 2 -- src/types.ts | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0ccc5d1f..5e23d13c 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,12 @@ There's a multitude of settings you can use to control the plugin server. Use th | CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | | PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | | PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | +| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | +| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | +| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | +| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | +| CLICKHOUSE_CA | ClickHouse CA certs | `null` | +| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | | KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | | KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | | KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | diff --git a/src/config.ts b/src/config.ts index d95bf4d4..594c61b3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -15,7 +15,6 @@ export function getDefaultConfig(): PluginsServerConfig { CLICKHOUSE_PASSWORD: null, CLICKHOUSE_CA: null, CLICKHOUSE_SECURE: false, - CLICKHOUSE_VERIFY: true, KAFKA_ENABLED: false, KAFKA_HOSTS: null, KAFKA_CLIENT_CERT_B64: null, @@ -50,7 +49,6 @@ export function getConfigHelp(): Record { CLICKHOUSE_PASSWORD: 'ClickHouse password', CLICKHOUSE_CA: 'ClickHouse CA certs', CLICKHOUSE_SECURE: 'Secure ClickHouse connection', - CLICKHOUSE_VERIFY: 'Verify ClickHouse connection', REDIS_URL: 'Redis store URL', BASE_DIR: 'base path for resolving local plugins', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'Redis channel for reload events', diff --git a/src/types.ts b/src/types.ts index 57a2deb8..303f06ff 100644 --- a/src/types.ts +++ b/src/types.ts @@ -28,7 +28,6 @@ export interface PluginsServerConfig extends Record { CLICKHOUSE_PASSWORD: string | null CLICKHOUSE_CA: string | null CLICKHOUSE_SECURE: boolean - CLICKHOUSE_VERIFY: boolean KAFKA_ENABLED: boolean KAFKA_HOSTS: string | null KAFKA_CLIENT_CERT_B64: string | null From 49bbf72545b8fc99c61d025880ca2f9d2df993db Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 19 Jan 2021 18:02:28 +0100 Subject: [PATCH 096/218] Fix sanitizeEventName --- src/ingestion/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index c6eb0df4..1f78235e 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -64,5 +64,5 @@ export function sanitizeEventName(eventName: any): string { eventName = String(eventName) } } - return eventName.substr(0, 2) + return eventName.substr(0, 200) } From 8dca9a36d984d139c342cf0abf9b7600e6a58e70 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 02:19:37 +0100 Subject: [PATCH 097/218] Await queue start --- src/ingestion/kafka-queue.ts | 6 ++++++ src/server.ts | 4 ++-- src/worker/queue.ts | 26 ++++++++++++++------------ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index a1459a50..5fc3dd25 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -30,6 +30,10 @@ export class KafkaQueue implements Queue { } async start(): Promise { + const promise = new Promise((resolve, reject) => { + this.consumer.on(this.consumer.events.GROUP_JOIN, () => resolve()) + this.consumer.on(this.consumer.events.CRASH, ({ payload: { error } }) => reject(error)) + }) status.info('โฌ', `Connecting Kafka consumer to ${this.pluginsServer.KAFKA_HOSTS}...`) await this.consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) // KafkaJS batching: https://kafka.js.org/docs/consuming#a-name-each-batch-a-eachbatch @@ -80,6 +84,7 @@ export class KafkaQueue implements Queue { }, }) this.wasConsumerRan = true + return promise } async pause(): Promise { @@ -126,6 +131,7 @@ export class KafkaQueue implements Queue { status.error('โš ๏ธ', `Kafka consumer group ${groupId} crashed!`) console.error(error) Sentry.captureException(error) + process.kill(process.pid, 'SIGINT') }) consumer.on(CONNECT, () => { status.info('โœ…', 'Kafka consumer connected!') diff --git a/src/server.ts b/src/server.ts index e6bcb9f9..e62a6459 100644 --- a/src/server.ts +++ b/src/server.ts @@ -222,7 +222,7 @@ export async function startPluginsServer( fastifyInstance = await startFastifyInstance(server) } - queue = startQueue(server, processEvent, processEventBatch) + queue = await startQueue(server, processEvent, processEventBatch) piscina.on('drain', () => { queue?.resume() }) @@ -236,7 +236,7 @@ export async function startPluginsServer( await stopSchedule?.() await stopPiscina(piscina!) piscina = makePiscina(serverConfig!) - queue = startQueue(server!, processEvent, processEventBatch) + queue = await startQueue(server!, processEvent, processEventBatch) stopSchedule = await startSchedule(server!, piscina) } }) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 9e99e532..377520c5 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -5,23 +5,28 @@ import Worker from '../celery/worker' import Client from '../celery/client' import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' -import { UUID } from '../utils' import { status } from '../status' -export function startQueue( +export async function startQueue( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Queue { +): Promise { const relevantStartQueue = server.KAFKA_ENABLED ? startQueueKafka : startQueueRedis - return relevantStartQueue(server, processEvent, processEventBatch) + try { + return await relevantStartQueue(server, processEvent, processEventBatch) + } catch (error) { + status.error('๐Ÿ’ฅ', `Failed to start Kafka queue:\n${error}`) + process.kill(process.pid, 'SIGINT') + process.exit(1) + } } -function startQueueRedis( +async function startQueueRedis( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Queue { +): Promise { const worker = new Worker(server.redis, server.PLUGINS_CELERY_QUEUE) const client = new Client(server.redis, server.CELERY_DEFAULT_QUEUE) @@ -65,11 +70,11 @@ function startQueueRedis( return worker } -function startQueueKafka( +async function startQueueKafka( server: PluginsServer, processEvent: (event: PluginEvent) => Promise, processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> -): Queue { +): Promise { const kafkaQueue = new KafkaQueue(server, processEventBatch, async (event: PluginEvent) => { const singleIngestionTimer = new Date() const { distinct_id, ip, site_url, team_id, now, sent_at, uuid } = event @@ -86,10 +91,7 @@ function startQueueKafka( server.statsd?.timing('single-ingestion', singleIngestionTimer) }) - kafkaQueue.start().catch((reason) => { - status.error('๐Ÿ’ฅ', `Failed to start Kafka queue:\n${reason}`) - process.exit(1) - }) + await kafkaQueue.start() return kafkaQueue } From 281e5d4a488deac153c1c0498a64255b221d9eb5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 10:45:45 +0100 Subject: [PATCH 098/218] Prettier README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5e23d13c..40f65124 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,12 @@ There's a multitude of settings you can use to control the plugin server. Use th | CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | | PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | | PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | -| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | -| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | -| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | -| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | -| CLICKHOUSE_CA | ClickHouse CA certs | `null` | -| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | +| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | +| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | +| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | +| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | +| CLICKHOUSE_CA | ClickHouse CA certs | `null` | +| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | | KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | | KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | | KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | From f153e87efba108082f855b30de22f54b3886b086 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 13:03:36 +0100 Subject: [PATCH 099/218] Try ClickHouse CI skeleton --- .github/workflows/ci.yml | 81 ++++++++++++++++++++++++++++++++++++++-- src/config.ts | 2 +- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4364a3a6..9d4997ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,83 @@ jobs: - name: Lint with ESLint run: yarn lint + tests-clickhouse: + name: Tests โ€“ ClickHouse + Kafka + runs-on: ubuntu-20.04 + + services: + postgres: + image: postgres:12 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + clickhouse: + image: yandex/clickhouse-server + ports: + - '8123:8123' + - '9000:9000' + - '9440:9440' + - '9009:9009' + zookeeper: + image: wurstmeister/zookeeper + kafka: + image: wurstmeister/kafka + ports: + - '9092:9092' + env: + KAFKA_ADVERTISED_HOST_NAME: kafka + sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + env: + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + PRIMARY_DB: 'clickhouse' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'posthog_test' + CLICKHOUSE_SECURE: 'false' + CLICKHOUSE_VERIFY: 'false' + TEST: 'true' + + steps: + + - name: Check out Django server for database setup + uses: actions/checkout@v2 + with: + repository: 'PostHog/posthog' + ref: 'fix-ch-compose' + path: 'django/' + + - name: Install requirements.txt dependencies with pip + run: | + python -m pip install --upgrade pip + python -m pip install -r django/requirements.txt + + - name: Set up ClickHouse + run: | + python django/manage.py clickhouse_test_setup + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check out plugin server + uses: actions/checkout@v2 + + - name: Set up Node 14 + uses: actions/setup-node@v2 + with: + node-version: 14 + + - name: Tear down ClickHouse + run: | + python django/manage.py clickhouse_test_teardown + tests: - name: Tests + name: Tests โ€“ Postgres + Redis runs-on: ubuntu-20.04 services: @@ -51,10 +126,10 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Node 14 - uses: actions/setup-node@v1 + uses: actions/setup-node@v2 with: node-version: 14 diff --git a/src/config.ts b/src/config.ts index 594c61b3..28d5ca8d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,7 +10,7 @@ export function getDefaultConfig(): PluginsServerConfig { CELERY_DEFAULT_QUEUE: 'celery', DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/posthog_test' : 'postgres://localhost:5432/posthog', CLICKHOUSE_HOST: 'localhost', - CLICKHOUSE_DATABASE: process.env.NODE_ENV === 'test' ? 'posthog_test' : 'default', + CLICKHOUSE_DATABASE: 'default', CLICKHOUSE_USERNAME: 'default', CLICKHOUSE_PASSWORD: null, CLICKHOUSE_CA: null, From bdf977086b99515f2d62d2199a8cce8d4d16b3fa Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 13:36:07 +0100 Subject: [PATCH 100/218] Update yarn.lock --- yarn.lock | 235 +++++++++++++++++++++--------------------------------- 1 file changed, 90 insertions(+), 145 deletions(-) diff --git a/yarn.lock b/yarn.lock index f4dce87f..18bc78fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -888,10 +888,10 @@ exec-sh "^0.3.2" minimist "^1.2.0" -"@eslint/eslintrc@^0.2.2": - version "0.2.2" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.2.2.tgz#d01fc791e2fc33e88a29d6f3dc7e93d0cd784b76" - integrity sha512-EfB5OHNYp1F4px/LI/FEnGylop7nOqkQ1LRzCM0KccA2U8tvV8w01KBv37LbO7nW4H+YhKyo2LcJhRwjjV17QQ== +"@eslint/eslintrc@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.3.0.tgz#d736d6963d7003b6514e6324bec9c602ac340318" + integrity sha512-1JTKgrOKAHVivSvOYw+sJOunkBjUOvjqWk1DPja7ZFhIS2mX/4EgTT8M7eTK9jrKhL/FvXXEbQwIs3pg1xp3dg== dependencies: ajv "^6.12.4" debug "^4.1.1" @@ -900,7 +900,7 @@ ignore "^4.0.6" import-fresh "^3.2.1" js-yaml "^3.13.1" - lodash "^4.17.19" + lodash "^4.17.20" minimatch "^3.0.4" strip-json-comments "^3.1.1" @@ -1549,61 +1549,62 @@ dependencies: "@types/yargs-parser" "*" -"@typescript-eslint/eslint-plugin@^4.6.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.9.1.tgz#66758cbe129b965fe9c63b04b405d0cf5280868b" - integrity sha512-QRLDSvIPeI1pz5tVuurD+cStNR4sle4avtHhxA+2uyixWGFjKzJ+EaFVRW6dA/jOgjV5DTAjOxboQkRDE8cRlQ== +"@typescript-eslint/eslint-plugin@^4.14.0": + version "4.14.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.14.0.tgz#92db8e7c357ed7d69632d6843ca70b71be3a721d" + integrity sha512-IJ5e2W7uFNfg4qh9eHkHRUCbgZ8VKtGwD07kannJvM5t/GU8P8+24NX8gi3Hf5jST5oWPY8kyV1s/WtfiZ4+Ww== dependencies: - "@typescript-eslint/experimental-utils" "4.9.1" - "@typescript-eslint/scope-manager" "4.9.1" + "@typescript-eslint/experimental-utils" "4.14.0" + "@typescript-eslint/scope-manager" "4.14.0" debug "^4.1.1" functional-red-black-tree "^1.0.1" + lodash "^4.17.15" regexpp "^3.0.0" semver "^7.3.2" tsutils "^3.17.1" -"@typescript-eslint/experimental-utils@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.9.1.tgz#86633e8395191d65786a808dc3df030a55267ae2" - integrity sha512-c3k/xJqk0exLFs+cWSJxIjqLYwdHCuLWhnpnikmPQD2+NGAx9KjLYlBDcSI81EArh9FDYSL6dslAUSwILeWOxg== +"@typescript-eslint/experimental-utils@4.14.0": + version "4.14.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.14.0.tgz#5aa7b006736634f588a69ee343ca959cd09988df" + integrity sha512-6i6eAoiPlXMKRbXzvoQD5Yn9L7k9ezzGRvzC/x1V3650rUk3c3AOjQyGYyF9BDxQQDK2ElmKOZRD0CbtdkMzQQ== dependencies: "@types/json-schema" "^7.0.3" - "@typescript-eslint/scope-manager" "4.9.1" - "@typescript-eslint/types" "4.9.1" - "@typescript-eslint/typescript-estree" "4.9.1" + "@typescript-eslint/scope-manager" "4.14.0" + "@typescript-eslint/types" "4.14.0" + "@typescript-eslint/typescript-estree" "4.14.0" eslint-scope "^5.0.0" eslint-utils "^2.0.0" -"@typescript-eslint/parser@^4.6.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.9.1.tgz#2d74c4db5dd5117379a9659081a4d1ec02629055" - integrity sha512-Gv2VpqiomvQ2v4UL+dXlQcZ8zCX4eTkoIW+1aGVWT6yTO+6jbxsw7yQl2z2pPl/4B9qa5JXeIbhJpONKjXIy3g== +"@typescript-eslint/parser@^4.14.0": + version "4.14.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.14.0.tgz#62d4cd2079d5c06683e9bfb200c758f292c4dee7" + integrity sha512-sUDeuCjBU+ZF3Lzw0hphTyScmDDJ5QVkyE21pRoBo8iDl7WBtVFS+WDN3blY1CH3SBt7EmYCw6wfmJjF0l/uYg== dependencies: - "@typescript-eslint/scope-manager" "4.9.1" - "@typescript-eslint/types" "4.9.1" - "@typescript-eslint/typescript-estree" "4.9.1" + "@typescript-eslint/scope-manager" "4.14.0" + "@typescript-eslint/types" "4.14.0" + "@typescript-eslint/typescript-estree" "4.14.0" debug "^4.1.1" -"@typescript-eslint/scope-manager@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.9.1.tgz#cc2fde310b3f3deafe8436a924e784eaab265103" - integrity sha512-sa4L9yUfD/1sg9Kl8OxPxvpUcqxKXRjBeZxBuZSSV1v13hjfEJkn84n0An2hN8oLQ1PmEl2uA6FkI07idXeFgQ== +"@typescript-eslint/scope-manager@4.14.0": + version "4.14.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.14.0.tgz#55a4743095d684e1f7b7180c4bac2a0a3727f517" + integrity sha512-/J+LlRMdbPh4RdL4hfP1eCwHN5bAhFAGOTsvE6SxsrM/47XQiPSgF5MDgLyp/i9kbZV9Lx80DW0OpPkzL+uf8Q== dependencies: - "@typescript-eslint/types" "4.9.1" - "@typescript-eslint/visitor-keys" "4.9.1" + "@typescript-eslint/types" "4.14.0" + "@typescript-eslint/visitor-keys" "4.14.0" -"@typescript-eslint/types@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.9.1.tgz#a1a7dd80e4e5ac2c593bc458d75dd1edaf77faa2" - integrity sha512-fjkT+tXR13ks6Le7JiEdagnwEFc49IkOyys7ueWQ4O8k4quKPwPJudrwlVOJCUQhXo45PrfIvIarcrEjFTNwUA== +"@typescript-eslint/types@4.14.0": + version "4.14.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.14.0.tgz#d8a8202d9b58831d6fd9cee2ba12f8a5a5dd44b6" + integrity sha512-VsQE4VvpldHrTFuVPY1ZnHn/Txw6cZGjL48e+iBxTi2ksa9DmebKjAeFmTVAYoSkTk7gjA7UqJ7pIsyifTsI4A== -"@typescript-eslint/typescript-estree@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.9.1.tgz#6e5b86ff5a5f66809e1f347469fadeec69ac50bf" - integrity sha512-bzP8vqwX6Vgmvs81bPtCkLtM/Skh36NE6unu6tsDeU/ZFoYthlTXbBmpIrvosgiDKlWTfb2ZpPELHH89aQjeQw== +"@typescript-eslint/typescript-estree@4.14.0": + version "4.14.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.14.0.tgz#4bcd67486e9acafc3d0c982b23a9ab8ac8911ed7" + integrity sha512-wRjZ5qLao+bvS2F7pX4qi2oLcOONIB+ru8RGBieDptq/SudYwshveORwCVU4/yMAd4GK7Fsf8Uq1tjV838erag== dependencies: - "@typescript-eslint/types" "4.9.1" - "@typescript-eslint/visitor-keys" "4.9.1" + "@typescript-eslint/types" "4.14.0" + "@typescript-eslint/visitor-keys" "4.14.0" debug "^4.1.1" globby "^11.0.1" is-glob "^4.0.1" @@ -1611,12 +1612,12 @@ semver "^7.3.2" tsutils "^3.17.1" -"@typescript-eslint/visitor-keys@4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.9.1.tgz#d76374a58c4ead9e92b454d186fea63487b25ae1" - integrity sha512-9gspzc6UqLQHd7lXQS7oWs+hrYggspv/rk6zzEMhCbYwPE/sF7oxo7GAjkS35Tdlt7wguIG+ViWCPtVZHz/ybQ== +"@typescript-eslint/visitor-keys@4.14.0": + version "4.14.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.14.0.tgz#b1090d9d2955b044b2ea2904a22496849acbdf54" + integrity sha512-MeHHzUyRI50DuiPgV9+LxcM52FCJFYjJiWHtXlbyC27b80mfOwKeiKI+MHOTEpcpfmoPFm/vvQS88bYIx6PZTA== dependencies: - "@typescript-eslint/types" "4.9.1" + "@typescript-eslint/types" "4.14.0" eslint-visitor-keys "^2.0.0" JSONStream@1.3.4: @@ -1687,7 +1688,7 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" -ajv@^6.10.0, ajv@^6.10.2, ajv@^6.11.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4: +ajv@^6.10.0, ajv@^6.11.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -1697,6 +1698,16 @@ ajv@^6.10.0, ajv@^6.10.2, ajv@^6.11.0, ajv@^6.12.2, ajv@^6.12.3, ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" +ajv@^7.0.2: + version "7.0.3" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-7.0.3.tgz#13ae747eff125cafb230ac504b2406cf371eece2" + integrity sha512-R50QRlXSxqXcQP5SvKUrw8VZeypvo12i2IX0EeR5PiZ7bEKeHWgzgo264LDadUsCU42lTJVhFikTqJwNeH34gQ== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + ansi-colors@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" @@ -1709,17 +1720,12 @@ ansi-escapes@^4.2.1, ansi-escapes@^4.3.0: dependencies: type-fest "^0.11.0" -ansi-regex@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" - integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== - ansi-regex@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== -ansi-styles@^3.2.0, ansi-styles@^3.2.1: +ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== @@ -1853,11 +1859,6 @@ assign-symbols@^1.0.0: resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" integrity sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c= -astral-regex@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-1.0.0.tgz#6c8c3fb827dd43ee3918f27b82782ab7658a6fd9" - integrity sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg== - astral-regex@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" @@ -2720,11 +2721,6 @@ emittery@^0.7.1: resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.7.2.tgz#25595908e13af0f5674ab419396e2fb394cdfa82" integrity sha512-A8OG5SR/ij3SsJdWDJdkkSYUjQdCUx6APQXem0SaEePBSRg4eymGYwBkKo1Y6DU+af/Jn2dBQqDBvjnr9Vi8nQ== -emoji-regex@^7.0.1: - version "7.0.3" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" - integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA== - emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" @@ -2884,12 +2880,10 @@ escodegen@^1.14.1: optionalDependencies: source-map "~0.6.1" -eslint-config-prettier@^6.9.0: - version "6.15.0" - resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-6.15.0.tgz#7f93f6cb7d45a92f1537a70ecc06366e1ac6fed9" - integrity sha512-a1+kOYLR8wMGustcgAjdydMsQ2A/2ipRPwRKUmfYaSxc9ZPcrku080Ctl6zrZzZNs/U82MjSv+qKREkoq3bJaw== - dependencies: - get-stdin "^6.0.0" +eslint-config-prettier@^7.2.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-7.2.0.tgz#f4a4bd2832e810e8cc7c1411ec85b3e85c0c53f9" + integrity sha512-rV4Qu0C3nfJKPOAhFujFxB7RMP+URFyQqqOZW9DMRD7ZDTFyjaIlETU3xzHELt++4ugC0+Jm084HQYkkJe+Ivg== eslint-import-resolver-node@^0.3.4: version "0.3.4" @@ -2946,13 +2940,6 @@ eslint-plugin-node@^11.1.0: resolve "^1.10.1" semver "^6.1.0" -eslint-plugin-prettier@^3.1.2: - version "3.2.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-3.2.0.tgz#af391b2226fa0e15c96f36c733f6e9035dbd952c" - integrity sha512-kOUSJnFjAUFKwVxuzy6sA5yyMx6+o9ino4gCdShzBNx4eyFRudWRYKCFolKjoM40PEiuU6Cn7wBLfq3WsGg7qg== - dependencies: - prettier-linter-helpers "^1.0.0" - eslint-plugin-promise@^4.1.1: version "4.2.1" resolved "https://registry.yarnpkg.com/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz#845fd8b2260ad8f82564c1222fce44ad71d9418a" @@ -2975,11 +2962,6 @@ eslint-plugin-react@^7.20.0: resolve "^1.18.1" string.prototype.matchall "^4.0.2" -eslint-plugin-standard@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz#0c3bf3a67e853f8bbbc580fb4945fbf16f41b7c5" - integrity sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ== - eslint-scope@^5.0.0, eslint-scope@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c" @@ -3005,13 +2987,13 @@ eslint-visitor-keys@^2.0.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz#21fdc8fbcd9c795cc0321f0563702095751511a8" integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== -eslint@^7.0.0: - version "7.15.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.15.0.tgz#eb155fb8ed0865fcf5d903f76be2e5b6cd7e0bc7" - integrity sha512-Vr64xFDT8w30wFll643e7cGrIkPEU50yIiI36OdSIDoSGguIeaLzBo0vpGvzo9RECUqq7htURfwEtKqwytkqzA== +eslint@^7.18.0: + version "7.18.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.18.0.tgz#7fdcd2f3715a41fe6295a16234bd69aed2c75e67" + integrity sha512-fbgTiE8BfUJZuBeq2Yi7J3RB3WGUQ9PNuNbmgi6jt9Iv8qrkxfy19Ds3OpL1Pm7zg3BtTVhvcUZbIRQ0wmSjAQ== dependencies: "@babel/code-frame" "^7.0.0" - "@eslint/eslintrc" "^0.2.2" + "@eslint/eslintrc" "^0.3.0" ajv "^6.10.0" chalk "^4.0.0" cross-spawn "^7.0.2" @@ -3035,7 +3017,7 @@ eslint@^7.0.0: js-yaml "^3.13.1" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" - lodash "^4.17.19" + lodash "^4.17.20" minimatch "^3.0.4" natural-compare "^1.4.0" optionator "^0.9.1" @@ -3044,7 +3026,7 @@ eslint@^7.0.0: semver "^7.2.1" strip-ansi "^6.0.0" strip-json-comments "^3.1.0" - table "^5.2.3" + table "^6.0.4" text-table "^0.2.0" v8-compile-cache "^2.0.3" @@ -3233,11 +3215,6 @@ fast-deep-equal@^3.1.1: resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== -fast-diff@^1.1.2: - version "1.2.0" - resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.2.0.tgz#73ee11982d86caaf7959828d519cfe927fac5f03" - integrity sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w== - fast-glob@^3.1.1: version "3.2.4" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.4.tgz#d20aefbf99579383e7f3cc66529158c9b98554d3" @@ -3570,11 +3547,6 @@ get-stdin@^4.0.1: resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-4.0.1.tgz#b968c6b0a04384324902e8bf1a5df32579a450fe" integrity sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4= -get-stdin@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-6.0.0.tgz#9e09bf712b360ab9225e812048f71fde9c89657b" - integrity sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g== - get-stream@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" @@ -4083,11 +4055,6 @@ is-finite@^1.0.0: resolved "https://registry.yarnpkg.com/is-finite/-/is-finite-1.1.0.tgz#904135c77fb42c0641d6aa1bcdbc4daa8da082f3" integrity sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w== -is-fullwidth-code-point@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" - integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8= - is-fullwidth-code-point@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" @@ -4732,6 +4699,11 @@ json-schema-traverse@^0.4.1: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== +json-schema-traverse@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" + integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== + json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -4968,7 +4940,7 @@ lodash@4.17.19: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b" integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ== -lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20: +lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20: version "4.17.20" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== @@ -5838,14 +5810,7 @@ prelude-ls@~1.1.2: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= -prettier-linter-helpers@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz#d23d41fe1375646de2d0104d3454a3008802cf7b" - integrity sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w== - dependencies: - fast-diff "^1.1.2" - -prettier@^2.1.2: +prettier@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.2.1.tgz#795a1a78dd52f073da0cd42b21f9c91381923ff5" integrity sha512-PqyhM2yCjg/oKkFPtTGUojv7gnZAoG80ttl45O6x2Ug/rMJw4wcc9k6aaf2hibP7BGVCCM33gZoGjyvt9mm16Q== @@ -6264,6 +6229,11 @@ require-directory@^2.1.1: resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= +require-from-string@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" + integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== + require-main-filename@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" @@ -6535,15 +6505,6 @@ slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -slice-ansi@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-2.1.0.tgz#cacd7693461a637a5788d92a7dd4fba068e81636" - integrity sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ== - dependencies: - ansi-styles "^3.2.0" - astral-regex "^1.0.0" - is-fullwidth-code-point "^2.0.0" - slice-ansi@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" @@ -6769,15 +6730,6 @@ string-similarity@^4.0.1: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.3.tgz#ef52d6fc59c8a0fc93b6307fbbc08cc6e18cde21" integrity sha512-QEwJzNFCqq+5AGImk5z4vbsEPTN/+gtyKfXBVLBcbPBRPNganZGfQnIuf9yJ+GiwSnD65sT8xrw/uwU1Q1WmfQ== -string-width@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961" - integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w== - dependencies: - emoji-regex "^7.0.1" - is-fullwidth-code-point "^2.0.0" - strip-ansi "^5.1.0" - string-width@^4.1.0, string-width@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5" @@ -6839,13 +6791,6 @@ stringify-object@^3.3.0: is-obj "^1.0.1" is-regexp "^1.0.0" -strip-ansi@^5.1.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae" - integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA== - dependencies: - ansi-regex "^4.1.0" - strip-ansi@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" @@ -6954,15 +6899,15 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== -table@^5.2.3: - version "5.4.6" - resolved "https://registry.yarnpkg.com/table/-/table-5.4.6.tgz#1292d19500ce3f86053b05f0e8e7e4a3bb21079e" - integrity sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug== +table@^6.0.4: + version "6.0.7" + resolved "https://registry.yarnpkg.com/table/-/table-6.0.7.tgz#e45897ffbcc1bcf9e8a87bf420f2c9e5a7a52a34" + integrity sha512-rxZevLGTUzWna/qBLObOe16kB2RTnnbhciwgPbMMlazz1yZGVEgnZK762xyVdVznhqxrfCeBMmMkgOOaPwjH7g== dependencies: - ajv "^6.10.2" - lodash "^4.17.14" - slice-ansi "^2.1.0" - string-width "^3.0.0" + ajv "^7.0.2" + lodash "^4.17.20" + slice-ansi "^4.0.0" + string-width "^4.2.0" tar-stream@^2.1.4: version "2.1.4" From c0a2e4103284a4c457d69939f000ea3be1e92efe Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 13:44:45 +0100 Subject: [PATCH 101/218] Possibly fix CI --- .github/workflows/ci.yml | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d4997ae..4a6fc1e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,18 +55,7 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - env: - SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' - PRIMARY_DB: 'clickhouse' - CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'posthog_test' - CLICKHOUSE_SECURE: 'false' - CLICKHOUSE_VERIFY: 'false' - TEST: 'true' - steps: - - name: Check out Django server for database setup uses: actions/checkout@v2 with: @@ -74,20 +63,29 @@ jobs: ref: 'fix-ch-compose' path: 'django/' + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install requirements.txt dependencies with pip run: | python -m pip install --upgrade pip python -m pip install -r django/requirements.txt - name: Set up ClickHouse + env: + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + PRIMARY_DB: 'clickhouse' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'posthog_test' + CLICKHOUSE_SECURE: 'false' + CLICKHOUSE_VERIFY: 'false' + TEST: 'true' run: | python django/manage.py clickhouse_test_setup - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Check out plugin server uses: actions/checkout@v2 @@ -97,10 +95,19 @@ jobs: node-version: 14 - name: Tear down ClickHouse + env: + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + PRIMARY_DB: 'clickhouse' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'posthog_test' + CLICKHOUSE_SECURE: 'false' + CLICKHOUSE_VERIFY: 'false' + TEST: 'true' run: | python django/manage.py clickhouse_test_teardown - tests: + tests-postgres: name: Tests โ€“ Postgres + Redis runs-on: ubuntu-20.04 From a4474b59b7c0912de781b2e90ec619894ea7058f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 13:46:13 +0100 Subject: [PATCH 102/218] Revert basic tests job to old name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a6fc1e4..13dcd551 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,8 +107,8 @@ jobs: run: | python django/manage.py clickhouse_test_teardown - tests-postgres: - name: Tests โ€“ Postgres + Redis + tests: + name: Tests runs-on: ubuntu-20.04 services: From 0656a6862bd876b0544f28696cde452c8b0cbf7e Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 13:50:58 +0100 Subject: [PATCH 103/218] Cache pip install --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13dcd551..d6bd5530 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,16 @@ jobs: with: python-version: 3.8 + - uses: actions/cache@v2 + id: pip + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install requirements.txt dependencies with pip + if: steps.pip.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip python -m pip install -r django/requirements.txt From 314de740f6ffa87e93de62c8bcb8ad32af477b72 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 13:59:01 +0100 Subject: [PATCH 104/218] Fix ClickHouse teardown and add Yarn installation --- .github/workflows/ci.yml | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6bd5530..fd81ca33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,14 +60,24 @@ jobs: uses: actions/checkout@v2 with: repository: 'PostHog/posthog' - ref: 'fix-ch-compose' - path: 'django/' + ref: 'fix-ch-compose' # TODO: remove this line to use master + path: 'posthog/' + + - name: Check out plugin server + uses: actions/checkout@v2 + with: + path: 'plugin-server' - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.8 + - name: Set up Node 14 + uses: actions/setup-node@v2 + with: + node-version: 14 + - uses: actions/cache@v2 id: pip with: @@ -80,7 +90,7 @@ jobs: if: steps.pip.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip - python -m pip install -r django/requirements.txt + python -m pip install -r posthog/requirements.txt - name: Set up ClickHouse env: @@ -93,15 +103,10 @@ jobs: CLICKHOUSE_VERIFY: 'false' TEST: 'true' run: | - python django/manage.py clickhouse_test_setup + python posthog/manage.py clickhouse_test_setup - - name: Check out plugin server - uses: actions/checkout@v2 - - - name: Set up Node 14 - uses: actions/setup-node@v2 - with: - node-version: 14 + - name: Install package.json dependencies with Yarn + run: cd plugin-server && yarn - name: Tear down ClickHouse env: @@ -114,7 +119,7 @@ jobs: CLICKHOUSE_VERIFY: 'false' TEST: 'true' run: | - python django/manage.py clickhouse_test_teardown + python posthog/manage.py clickhouse_test_teardown tests: name: Tests From caf4d1375ce058e2a777b6866cef28ff573ef6d6 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 14:05:24 +0100 Subject: [PATCH 105/218] Await startQueue in tests --- tests/queue.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queue.test.ts b/tests/queue.test.ts index f79824b8..78b003d8 100644 --- a/tests/queue.test.ts +++ b/tests/queue.test.ts @@ -67,7 +67,7 @@ test('worker and task passing via redis', async () => { expect(args2).toEqual(args) expect(kwargs2).toEqual({}) - const queue = startQueue( + const queue = await startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) @@ -132,7 +132,7 @@ test('process multiple tasks', async () => { expect(await server.redis.llen(server.PLUGINS_CELERY_QUEUE)).toBe(3) expect(await server.redis.llen(server.CELERY_DEFAULT_QUEUE)).toBe(0) - const queue = startQueue( + const queue = await startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) @@ -199,7 +199,7 @@ test('pause and resume queue', async () => { expect(await server.redis.llen(server.PLUGINS_CELERY_QUEUE)).toBe(6) expect(await server.redis.llen(server.CELERY_DEFAULT_QUEUE)).toBe(0) - const queue = startQueue( + const queue = await startQueue( server, (event) => runPlugins(server, event), (events) => Promise.all(events.map((event) => runPlugins(server, event))) From 201afc94b4d35913bd150c2a3eac0e864cfbc576 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 14:08:13 +0100 Subject: [PATCH 106/218] Try existing test in new workflow --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd81ca33..8f7a8e6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,12 @@ jobs: - name: Install package.json dependencies with Yarn run: cd plugin-server && yarn + - name: Test with Jest + env: + DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + REDIS_URL: 'redis://localhost' + run: cd plugin-server && yarn test + - name: Tear down ClickHouse env: SECRET_KEY: 'abcdef' # unsafe - for testing only From ec49e2c59411fdf4920436f1cea1ce28c5f968ed Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 14:10:02 +0100 Subject: [PATCH 107/218] Don't cache pip requirements --- .github/workflows/ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f7a8e6a..e0949de1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,16 +78,7 @@ jobs: with: node-version: 14 - - uses: actions/cache@v2 - id: pip - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install requirements.txt dependencies with pip - if: steps.pip.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip python -m pip install -r posthog/requirements.txt From e6418d93f5d613962b7875a5e95790e852d47529 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 14:18:33 +0100 Subject: [PATCH 108/218] Add Redis to CH CI --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0949de1..ad8f8e50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,17 @@ jobs: POSTGRES_DB: postgres ports: ['5432:5432'] options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis + ports: + # Maps port 6379 on service container to the host + # Needed because `redis` host is not discoverable for some reason + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 clickhouse: image: yandex/clickhouse-server ports: From 7ee02ab2eded0b77975f2e0388954020927f3c53 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 14:42:59 +0100 Subject: [PATCH 109/218] Make some env var changes --- .github/workflows/ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad8f8e50..1647686d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,8 @@ jobs: - name: Set up ClickHouse env: SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' + REDIS_URL: 'redis://localhost' PRIMARY_DB: 'clickhouse' CLICKHOUSE_HOST: 'localhost' CLICKHOUSE_DATABASE: 'posthog_test' @@ -112,14 +113,15 @@ jobs: - name: Test with Jest env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test - name: Tear down ClickHouse env: SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' + REDIS_URL: 'redis://localhost' PRIMARY_DB: 'clickhouse' CLICKHOUSE_HOST: 'localhost' CLICKHOUSE_DATABASE: 'posthog_test' @@ -167,13 +169,13 @@ jobs: - name: Initiate database env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' REDIS_URL: 'redis://localhost' run: yarn task:init-test-db - name: Test with Jest env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' REDIS_URL: 'redis://localhost' run: yarn test From 39f189965cc6396a6c7859f7a5c53b80e8f67ab1 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 14:49:51 +0100 Subject: [PATCH 110/218] Try test_default as Django test DB name --- .github/workflows/ci.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1647686d..cb9777d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,7 @@ jobs: redis: image: redis ports: - # Maps port 6379 on service container to the host - # Needed because `redis` host is not discoverable for some reason - - 6379:6379 + - '6379:6379' options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -90,9 +88,7 @@ jobs: node-version: 14 - name: Install requirements.txt dependencies with pip - run: | - python -m pip install --upgrade pip - python -m pip install -r posthog/requirements.txt + run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt - name: Set up ClickHouse env: @@ -105,8 +101,7 @@ jobs: CLICKHOUSE_SECURE: 'false' CLICKHOUSE_VERIFY: 'false' TEST: 'true' - run: | - python posthog/manage.py clickhouse_test_setup + run: python posthog/manage.py clickhouse_test_setup - name: Install package.json dependencies with Yarn run: cd plugin-server && yarn @@ -147,9 +142,7 @@ jobs: redis: image: redis ports: - # Maps port 6379 on service container to the host - # Needed because `redis` host is not discoverable for some reason - - 6379:6379 + - '6379:6379' options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -195,9 +188,7 @@ jobs: redis: image: redis ports: - # Maps port 6379 on service container to the host - # Needed because `redis` host is not discoverable for some reason - - 6379:6379 + - '6379:6379' options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -217,12 +208,12 @@ jobs: - name: Initiate database env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' REDIS_URL: 'redis://localhost' run: yarn task:init-test-db - name: Run benchmarks env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' REDIS_URL: 'redis://localhost' run: yarn benchmark From 06b34dee17c139b4697bbce8bc6015a26d5a5281 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 14:55:04 +0100 Subject: [PATCH 111/218] Try test_posthog as Django test DB name --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb9777d9..8cf118aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres + POSTGRES_DB: test_posthog ports: ['5432:5432'] options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis: @@ -93,7 +93,7 @@ jobs: - name: Set up ClickHouse env: SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' PRIMARY_DB: 'clickhouse' CLICKHOUSE_HOST: 'localhost' @@ -108,14 +108,14 @@ jobs: - name: Test with Jest env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test - name: Tear down ClickHouse env: SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' PRIMARY_DB: 'clickhouse' CLICKHOUSE_HOST: 'localhost' From 7fa5f11b72cada78fd0f6da2d4b0849ea7573d8c Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:13:15 +0100 Subject: [PATCH 112/218] Don't tear down ClickHouse --- .github/workflows/ci.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cf118aa..dce5e18e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,11 +97,9 @@ jobs: REDIS_URL: 'redis://localhost' PRIMARY_DB: 'clickhouse' CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'posthog_test' - CLICKHOUSE_SECURE: 'false' - CLICKHOUSE_VERIFY: 'false' + CLICKHOUSE_DATABASE: 'test_posthog' TEST: 'true' - run: python posthog/manage.py clickhouse_test_setup + run: python posthog/manage.py setup_test_clickhouse - name: Install package.json dependencies with Yarn run: cd plugin-server && yarn @@ -112,20 +110,6 @@ jobs: REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test - - name: Tear down ClickHouse - env: - SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' - REDIS_URL: 'redis://localhost' - PRIMARY_DB: 'clickhouse' - CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'posthog_test' - CLICKHOUSE_SECURE: 'false' - CLICKHOUSE_VERIFY: 'false' - TEST: 'true' - run: | - python posthog/manage.py clickhouse_test_teardown - tests: name: Tests runs-on: ubuntu-20.04 From 07bfd2917fc18b6674649fdd196432ed5c8e6a37 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:14:13 +0100 Subject: [PATCH 113/218] Make env vars common to all tests-clickhouse steps --- .github/workflows/ci.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dce5e18e..1bbcf2a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,15 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + env: + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' + REDIS_URL: 'redis://localhost' + PRIMARY_DB: 'clickhouse' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'posthog_test' + TEST: 'true' + steps: - name: Check out Django server for database setup uses: actions/checkout@v2 @@ -91,14 +100,6 @@ jobs: run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt - name: Set up ClickHouse - env: - SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' - REDIS_URL: 'redis://localhost' - PRIMARY_DB: 'clickhouse' - CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'test_posthog' - TEST: 'true' run: python posthog/manage.py setup_test_clickhouse - name: Install package.json dependencies with Yarn From 0ab380b8ac1e7ee0a2359857b188a0b4e2b740ce Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:25:26 +0100 Subject: [PATCH 114/218] Debug Postgres --- tests/helpers/sql.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index c6b47756..0a9f1ef3 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -6,6 +6,8 @@ import { delay } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) const mocks = makePluginObjects(code) + console.log((await db.query('SELECT table_name FROM information_schema')).rows) + console.log((await db.query('SELECT datname FROM pg_database')).rows) await db.query('DELETE FROM posthog_pluginstorage') await db.query('DELETE FROM posthog_pluginattachment') await db.query('DELETE FROM posthog_pluginconfig') From e8d7960214088749c9a19b8ff819ba0de753a6c7 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:29:43 +0100 Subject: [PATCH 115/218] Debug Postgres actually --- tests/helpers/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 0a9f1ef3..7d1d8b89 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -6,7 +6,7 @@ import { delay } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) const mocks = makePluginObjects(code) - console.log((await db.query('SELECT table_name FROM information_schema')).rows) + console.log((await db.query('SELECT table_name FROM information_schema.tables')).rows) console.log((await db.query('SELECT datname FROM pg_database')).rows) await db.query('DELETE FROM posthog_pluginstorage') await db.query('DELETE FROM posthog_pluginattachment') From 111ff77d53606097179c2d23e16166047973df1a Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:34:16 +0100 Subject: [PATCH 116/218] Revert "Make env vars common to all tests-clickhouse steps" This reverts commit 07bfd2917fc18b6674649fdd196432ed5c8e6a37. --- .github/workflows/ci.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bbcf2a8..dce5e18e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,15 +64,6 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - env: - SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' - REDIS_URL: 'redis://localhost' - PRIMARY_DB: 'clickhouse' - CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'posthog_test' - TEST: 'true' - steps: - name: Check out Django server for database setup uses: actions/checkout@v2 @@ -100,6 +91,14 @@ jobs: run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt - name: Set up ClickHouse + env: + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' + REDIS_URL: 'redis://localhost' + PRIMARY_DB: 'clickhouse' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'test_posthog' + TEST: 'true' run: python posthog/manage.py setup_test_clickhouse - name: Install package.json dependencies with Yarn From e9cbf7d619866c44c01967a7898af88785faa206 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:34:39 +0100 Subject: [PATCH 117/218] Try fixing DB name discrepancy --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dce5e18e..cfc63717 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: - name: Set up ClickHouse env: SECRET_KEY: 'abcdef' # unsafe - for testing only - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' REDIS_URL: 'redis://localhost' PRIMARY_DB: 'clickhouse' CLICKHOUSE_HOST: 'localhost' From 3f5222a65526eeea8ed6a0e5ef7565bd1df65636 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:35:37 +0100 Subject: [PATCH 118/218] Make some env vars common --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc63717..253e6a7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,11 @@ jobs: KAFKA_ADVERTISED_HOST_NAME: kafka sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + env: + REDIS_URL: 'redis://localhost' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'test_posthog' + steps: - name: Check out Django server for database setup uses: actions/checkout@v2 @@ -94,10 +99,7 @@ jobs: env: SECRET_KEY: 'abcdef' # unsafe - for testing only DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' - REDIS_URL: 'redis://localhost' PRIMARY_DB: 'clickhouse' - CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'test_posthog' TEST: 'true' run: python posthog/manage.py setup_test_clickhouse From f0cf9165d68dbf795bac55726fbc10c9383e43ba Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:47:09 +0100 Subject: [PATCH 119/218] Debug Postgres better --- tests/helpers/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 7d1d8b89..e08db75c 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -6,7 +6,7 @@ import { delay } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) const mocks = makePluginObjects(code) - console.log((await db.query('SELECT table_name FROM information_schema.tables')).rows) + console.log((await db.query("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'")).rows) console.log((await db.query('SELECT datname FROM pg_database')).rows) await db.query('DELETE FROM posthog_pluginstorage') await db.query('DELETE FROM posthog_pluginattachment') From 974327da009c31e0d7063f4a4515d02bd0d5441f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:52:05 +0100 Subject: [PATCH 120/218] Remove Postgres debug code --- tests/helpers/sql.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index e08db75c..c6b47756 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -6,8 +6,6 @@ import { delay } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) const mocks = makePluginObjects(code) - console.log((await db.query("SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'")).rows) - console.log((await db.query('SELECT datname FROM pg_database')).rows) await db.query('DELETE FROM posthog_pluginstorage') await db.query('DELETE FROM posthog_pluginattachment') await db.query('DELETE FROM posthog_pluginconfig') From b8adacbc9411cf99df9b41285673b6634a521a01 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 15:58:19 +0100 Subject: [PATCH 121/218] Fix inserting --- tests/helpers/sql.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index c6b47756..13e3fecd 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -1,7 +1,7 @@ import { makePluginObjects } from './plugins' import { defaultConfig } from '../../src/config' import { Pool } from 'pg' -import { delay } from '../../src/utils' +import { delay, UUIDT } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) @@ -14,7 +14,24 @@ export async function resetTestDatabase(code: string): Promise { const team_ids = mocks.pluginConfigRows.map((c) => c.team_id) for (const team_id of team_ids) { - await insertRow(db, 'posthog_team', { id: team_id, name: 'TEST' }) + await insertRow(db, 'posthog_team', { + id: team_id, + name: 'TEST', + event_names: [], + event_names_with_usage: [], + event_properties: [], + event_properties_with_usage: [], + event_properties_numerical: [], + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + anonymize_ips: false, + completed_snippet_onboarding: true, + ingested_event: true, + uuid: new UUIDT().toString(), + session_recording_opt_in: true, + plugins_opt_in: true, + opt_out_capture: false, + }) } for (const plugin of mocks.pluginRows) { await insertRow(db, 'posthog_plugin', plugin) From 3fc0b8deca3a5a287ea9a0d96c2ae0498afad1bc Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 16:10:53 +0100 Subject: [PATCH 122/218] Fix posthog_team.app_urls --- tests/helpers/sql.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 13e3fecd..5ed3f0c8 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -16,6 +16,7 @@ export async function resetTestDatabase(code: string): Promise { for (const team_id of team_ids) { await insertRow(db, 'posthog_team', { id: team_id, + app_urls: [], name: 'TEST', event_names: [], event_names_with_usage: [], From 171a69188495404b78f4f500017ddf9ed24947d9 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 20 Jan 2021 16:27:21 +0100 Subject: [PATCH 123/218] Catch insertRow errors --- tests/helpers/sql.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 5ed3f0c8..67247e77 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -54,5 +54,11 @@ async function insertRow(db: Pool, table: string, object: Record): const params = Object.keys(object) .map((_, i) => `\$${i + 1}`) .join(',') - await db.query(`INSERT INTO ${table} (${keys}) VALUES (${params})`, Object.values(object)) + try { + await db.query(`INSERT INTO ${table} (${keys}) VALUES (${params})`, Object.values(object)) + } catch (error) { + console.error(`Error on table ${table} when inserting object:`) + console.error(object) + throw error + } } From 3f3f4404840cfd4404ca2f22eb90318682ea9360 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 08:47:19 +0100 Subject: [PATCH 124/218] Add plugin.organization_id in base tests --- tests/helpers/plugins.ts | 5 ++++- tests/helpers/sql.ts | 20 +++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/helpers/plugins.ts b/tests/helpers/plugins.ts index f6fe0f2b..0b0e33a5 100644 --- a/tests/helpers/plugins.ts +++ b/tests/helpers/plugins.ts @@ -4,8 +4,11 @@ import path from 'path' import os from 'os' import AdmZip from 'adm-zip' -export const plugin60: Plugin = { +export const organizationId = 'CA30F2EC-E9A4-4001-BF27-3EF194086068' + +export const plugin60: Plugin & { organization_id: string } = { id: 60, + organization_id: organizationId, plugin_type: 'custom', name: 'test-maxmind-plugin', description: 'Ingest GeoIP data via MaxMind', diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 67247e77..3e24ad18 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -1,4 +1,4 @@ -import { makePluginObjects } from './plugins' +import { makePluginObjects, organizationId } from './plugins' import { defaultConfig } from '../../src/config' import { Pool } from 'pg' import { delay, UUIDT } from '../../src/utils' @@ -12,12 +12,19 @@ export async function resetTestDatabase(code: string): Promise { await db.query('DELETE FROM posthog_plugin') await db.query('DELETE FROM posthog_team') - const team_ids = mocks.pluginConfigRows.map((c) => c.team_id) - for (const team_id of team_ids) { + const teamIds = mocks.pluginConfigRows.map((c) => c.team_id) + await insertRow(db, 'posthog_organization', { + id: organizationId, + name: 'TEST ORG', + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }) + for (const teamId of teamIds) { await insertRow(db, 'posthog_team', { - id: team_id, + id: teamId, + organization_id: organizationId, app_urls: [], - name: 'TEST', + name: 'TEST PROJECT', event_names: [], event_names_with_usage: [], event_properties: [], @@ -57,8 +64,7 @@ async function insertRow(db: Pool, table: string, object: Record): try { await db.query(`INSERT INTO ${table} (${keys}) VALUES (${params})`, Object.values(object)) } catch (error) { - console.error(`Error on table ${table} when inserting object:`) - console.error(object) + console.error(`Error on table ${table} when inserting object:\n`, object, '\n', error) throw error } } From d37c4855f568d6e581c81c6e5f9eeaee0e86714d Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 08:57:25 +0100 Subject: [PATCH 125/218] DELETE FROM posthog_organization --- tests/helpers/sql.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 3e24ad18..71db9c24 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -11,6 +11,7 @@ export async function resetTestDatabase(code: string): Promise { await db.query('DELETE FROM posthog_pluginconfig') await db.query('DELETE FROM posthog_plugin') await db.query('DELETE FROM posthog_team') + await db.query('DELETE FROM posthog_organization') const teamIds = mocks.pluginConfigRows.map((c) => c.team_id) await insertRow(db, 'posthog_organization', { From 59692a4430b7ae39a83bacebbaec3c777988d0f7 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 09:16:23 +0100 Subject: [PATCH 126/218] Rework CI to use Django everywhere --- .github/workflows/ci.yml | 143 +++++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 253e6a7b..6d06b724 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,8 @@ jobs: - name: Lint with ESLint run: yarn lint - tests-clickhouse: - name: Tests โ€“ ClickHouse + Kafka + tests: + name: Tests / Postgres + Redis runs-on: ubuntu-20.04 services: @@ -47,27 +47,9 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - clickhouse: - image: yandex/clickhouse-server - ports: - - '8123:8123' - - '9000:9000' - - '9440:9440' - - '9009:9009' - zookeeper: - image: wurstmeister/zookeeper - kafka: - image: wurstmeister/kafka - ports: - - '9092:9092' - env: - KAFKA_ADVERTISED_HOST_NAME: kafka - sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 env: REDIS_URL: 'redis://localhost' - CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'test_posthog' steps: - name: Check out Django server for database setup @@ -95,13 +77,12 @@ jobs: - name: Install requirements.txt dependencies with pip run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt - - name: Set up ClickHouse + - name: Set up databases env: SECRET_KEY: 'abcdef' # unsafe - for testing only DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' - PRIMARY_DB: 'clickhouse' TEST: 'true' - run: python posthog/manage.py setup_test_clickhouse + run: python posthog/manage.py setup_test_environment - name: Install package.json dependencies with Yarn run: cd plugin-server && yarn @@ -112,8 +93,8 @@ jobs: REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test - tests: - name: Tests + benchmarks: + name: Benchmarks / Postgres + Redis runs-on: ubuntu-20.04 services: @@ -122,7 +103,7 @@ jobs: env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres + POSTGRES_DB: test_posthog ports: ['5432:5432'] options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis: @@ -135,31 +116,53 @@ jobs: --health-timeout 5s --health-retries 5 + env: + REDIS_URL: 'redis://localhost' + steps: - - uses: actions/checkout@v2 + - name: Check out Django server for database setup + uses: actions/checkout@v2 + with: + repository: 'PostHog/posthog' + ref: 'fix-ch-compose' # TODO: remove this line to use master + path: 'posthog/' + + - name: Check out plugin server + uses: actions/checkout@v2 + with: + path: 'plugin-server' + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 - name: Set up Node 14 uses: actions/setup-node@v2 with: node-version: 14 - - name: Install package.json dependencies with Yarn - run: yarn + - name: Install requirements.txt dependencies with pip + run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt - - name: Initiate database + - name: Set up databases env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' - REDIS_URL: 'redis://localhost' - run: yarn task:init-test-db + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' + TEST: 'true' + run: python posthog/manage.py setup_test_environment + + - name: Install package.json dependencies with Yarn + run: cd plugin-server && yarn - name: Test with Jest env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' - run: yarn test + run: cd plugin-server && yarn benchmark - benchmarks: - name: Benchmarks + tests-clickhouse: + name: Tests / ClickHouse + Kafka runs-on: ubuntu-20.04 services: @@ -168,7 +171,7 @@ jobs: env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres + POSTGRES_DB: test_posthog ports: ['5432:5432'] options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis: @@ -180,26 +183,68 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + clickhouse: + image: yandex/clickhouse-server + ports: + - '8123:8123' + - '9000:9000' + - '9440:9440' + - '9009:9009' + zookeeper: + image: wurstmeister/zookeeper + kafka: + image: wurstmeister/kafka + ports: + - '9092:9092' + env: + KAFKA_ADVERTISED_HOST_NAME: kafka + sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + env: + REDIS_URL: 'redis://localhost' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'test_posthog' + KAFKA_ENABLED: 'true' steps: - - uses: actions/checkout@v1 + - name: Check out Django server for database setup + uses: actions/checkout@v2 + with: + repository: 'PostHog/posthog' + ref: 'fix-ch-compose' # TODO: remove this line to use master + path: 'posthog/' + + - name: Check out plugin server + uses: actions/checkout@v2 + with: + path: 'plugin-server' + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 - name: Set up Node 14 - uses: actions/setup-node@v1 + uses: actions/setup-node@v2 with: node-version: 14 - - name: Install package.json dependencies with Yarn - run: yarn + - name: Install requirements.txt dependencies with pip + run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt - - name: Initiate database + - name: Set up databases env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' - REDIS_URL: 'redis://localhost' - run: yarn task:init-test-db + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' + PRIMARY_DB: 'clickhouse' + TEST: 'true' + run: python posthog/manage.py setup_test_environment + + - name: Install package.json dependencies with Yarn + run: cd plugin-server && yarn - - name: Run benchmarks + - name: Test with Jest env: - DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres' + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' - run: yarn benchmark + run: cd plugin-server && yarn test From 4dff30e00eace2def548e9be0cde5f697e7c8263 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 09:20:09 +0100 Subject: [PATCH 127/218] Reorder tests for readability --- .github/workflows/ci.yml | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d06b724..ec949404 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,8 +93,8 @@ jobs: REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test - benchmarks: - name: Benchmarks / Postgres + Redis + tests-clickhouse: + name: Tests / ClickHouse + Kafka runs-on: ubuntu-20.04 services: @@ -115,9 +115,28 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + clickhouse: + image: yandex/clickhouse-server + ports: + - '8123:8123' + - '9000:9000' + - '9440:9440' + - '9009:9009' + zookeeper: + image: wurstmeister/zookeeper + kafka: + image: wurstmeister/kafka + ports: + - '9092:9092' + env: + KAFKA_ADVERTISED_HOST_NAME: kafka + sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 env: REDIS_URL: 'redis://localhost' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'test_posthog' + KAFKA_ENABLED: 'true' steps: - name: Check out Django server for database setup @@ -149,6 +168,7 @@ jobs: env: SECRET_KEY: 'abcdef' # unsafe - for testing only DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' + PRIMARY_DB: 'clickhouse' TEST: 'true' run: python posthog/manage.py setup_test_environment @@ -159,10 +179,10 @@ jobs: env: DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' - run: cd plugin-server && yarn benchmark + run: cd plugin-server && yarn test - tests-clickhouse: - name: Tests / ClickHouse + Kafka + benchmarks: + name: Benchmarks / Postgres + Redis runs-on: ubuntu-20.04 services: @@ -183,28 +203,9 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - clickhouse: - image: yandex/clickhouse-server - ports: - - '8123:8123' - - '9000:9000' - - '9440:9440' - - '9009:9009' - zookeeper: - image: wurstmeister/zookeeper - kafka: - image: wurstmeister/kafka - ports: - - '9092:9092' - env: - KAFKA_ADVERTISED_HOST_NAME: kafka - sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 env: REDIS_URL: 'redis://localhost' - CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'test_posthog' - KAFKA_ENABLED: 'true' steps: - name: Check out Django server for database setup @@ -236,7 +237,6 @@ jobs: env: SECRET_KEY: 'abcdef' # unsafe - for testing only DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' - PRIMARY_DB: 'clickhouse' TEST: 'true' run: python posthog/manage.py setup_test_environment @@ -247,4 +247,4 @@ jobs: env: DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' - run: cd plugin-server && yarn test + run: cd plugin-server && yarn benchmark From 1af923050ee69683aea047fcae451ee8eeddb5c3 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 10:00:55 +0100 Subject: [PATCH 128/218] Fix missing KAFKA_HOSTS in CI --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec949404..b7c494c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,7 @@ jobs: - name: Test with Jest env: + # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test @@ -130,13 +131,14 @@ jobs: - '9092:9092' env: KAFKA_ADVERTISED_HOST_NAME: kafka - sKAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 env: REDIS_URL: 'redis://localhost' CLICKHOUSE_HOST: 'localhost' CLICKHOUSE_DATABASE: 'test_posthog' KAFKA_ENABLED: 'true' + KAFKA_HOSTS: 'localhost:9092' steps: - name: Check out Django server for database setup @@ -177,6 +179,7 @@ jobs: - name: Test with Jest env: + # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test @@ -243,8 +246,9 @@ jobs: - name: Install package.json dependencies with Yarn run: cd plugin-server && yarn - - name: Test with Jest + - name: Run benchmarks env: + # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn benchmark From cebddb42fa502286151a7e7a2c703b55b7653367 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 10:36:58 +0100 Subject: [PATCH 129/218] Optimize createPosthog --- src/extensions/posthog.ts | 114 +++++++++++++++++++------------------- src/vm.ts | 22 ++++---- 2 files changed, 69 insertions(+), 67 deletions(-) diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index 2ff9dd0b..dd1ab85e 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -1,10 +1,20 @@ import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../ingestion/topics' +import { Properties } from '@posthog/plugin-scaffold' import { DateTime } from 'luxon' -import { PluginsServer, PluginConfig, RawEventMessage } from 'types' +import { PluginsServer, PluginConfig, RawEventMessage, EventData } from 'types' import { version } from '../../package.json' import Client from '../celery/client' import { UUIDT } from '../utils' +interface InternalData { + distinct_id: string + event: string + timestamp: string + properties: Properties + team_id: number + uuid: string +} + export interface DummyPostHog { capture(event: string, properties?: Record): void } @@ -12,68 +22,60 @@ export interface DummyPostHog { export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig): DummyPostHog { const distinctId = pluginConfig.plugin?.name || `plugin-id-${pluginConfig.plugin_id}` - const client = server.KAFKA_ENABLED ? null : new Client(server.redis, server.PLUGINS_CELERY_QUEUE) // Redis - const producer = server.KAFKA_ENABLED ? server.kafka!.producer() : null // Kafka - producer?.connect() + let sendEvent: (data: InternalData) => Promise - function sendEventRedis(event: string, properties: Record, timestamp: string) { - const data = { - distinct_id: distinctId, - event, - timestamp, - properties: { - $lib: 'posthog-plugin-server', - $lib_version: version, - ...properties, - }, + if (server.KAFKA_ENABLED) { + // Sending event to our Kafka>ClickHouse pipeline + const producer = server.kafka!.producer() + sendEvent = async (data) => { + await producer.connect() + producer!.send({ + topic: KAFKA_EVENTS_INGESTION_HANDOFF, + messages: [ + { + key: data.uuid, + value: JSON.stringify({ + distinct_id: data.distinct_id, + ip: '', + site_url: '', + data: JSON.stringify(data), + team_id: pluginConfig.team_id, + now: data.timestamp, + sent_at: data.timestamp, + uuid: data.uuid, + } as RawEventMessage), + }, + ], + }) } - - client!.sendTask( - 'posthog.tasks.process_event.process_event_with_plugins', - [distinctId, null, null, data, pluginConfig.team_id, timestamp, timestamp], - {} - ) - } - - function sendEventKafka(event: string, properties: Record, timestamp: string) { - const uuid = new UUIDT().toString() - const data = { - distinct_id: distinctId, - event, - timestamp, - properties: { - $lib: 'posthog-plugin-server', - $lib_version: version, - ...properties, - }, + } else { + // Sending event to our Redis>Postgres pipeline + const client = new Client(server.redis, server.PLUGINS_CELERY_QUEUE) + sendEvent = async (data) => { + client.sendTask( + 'posthog.tasks.process_event.process_event_with_plugins', + [data.distinct_id, null, null, data, pluginConfig.team_id, data.timestamp, data.timestamp], + {} + ) } - - producer!.send({ - topic: KAFKA_EVENTS_INGESTION_HANDOFF, - messages: [ - { - key: uuid, - value: JSON.stringify({ - distinct_id: distinctId, - ip: '', - site_url: '', - data: JSON.stringify(data), - team_id: pluginConfig.team_id, - now: timestamp, - sent_at: timestamp, - uuid, - } as RawEventMessage), - }, - ], - }) } - const sendEvent = server.KAFKA_ENABLED ? sendEventKafka : sendEventRedis - return { capture(event, properties = {}) { - const { timestamp, ...otherProperties } = properties - sendEvent(event, otherProperties, timestamp || DateTime.utc().toISO()) + const { timestamp = DateTime.utc().toISO(), ...otherProperties } = properties + const data: InternalData = { + distinct_id: distinctId, + event, + timestamp, + properties: { + $lib: 'posthog-plugin-server', + $lib_version: version, + ...otherProperties, + }, + team_id: pluginConfig.team_id, + uuid: new UUIDT().toString(), + } + sendEvent(data) }, } } diff --git a/src/vm.ts b/src/vm.ts index a1ab80ba..b1df874c 100644 --- a/src/vm.ts +++ b/src/vm.ts @@ -46,29 +46,29 @@ export async function createPluginConfigVM( // two ways packages could export themselves (plus "global") const module = { exports: {} }; let exports = {}; - + // helpers to get globals const __getExportDestinations = () => [exports, module.exports, global] const __getExported = (key) => __getExportDestinations().find(a => a[key])?.[key]; - - // the plugin JS code + + // the plugin JS code ${libJs}; ${indexJs}; // inject the meta object + shareable 'global' to the end of each exported function - const __pluginMeta = { - ...__pluginHostMeta, + const __pluginMeta = { + ...__pluginHostMeta, global: {} }; function __bindMeta (keyOrFunc) { const func = typeof keyOrFunc === 'function' ? keyOrFunc : __getExported(keyOrFunc); - if (func) return (...args) => func(...args, __pluginMeta); + if (func) return (...args) => func(...args, __pluginMeta); } function __callWithMeta (keyOrFunc, ...args) { const func = __bindMeta(keyOrFunc); - if (func) return func(...args); + if (func) return func(...args); } - + // we have processEvent, but not processEventBatch if (!__getExported('processEventBatch') && __getExported('processEvent')) { exports.processEventBatch = async function processEventBatch (batch, meta) { @@ -90,13 +90,13 @@ export async function createPluginConfigVM( return (await (__getExported('processEventBatch'))([event], meta))?.[0] } } - + // export various functions const __methods = { processEvent: __bindMeta('processEvent'), processEventBatch: __bindMeta('processEventBatch') }; - + // gather the runEveryX commands and export in __tasks const __tasks = {}; for (const exportDestination of __getExportDestinations().reverse()) { @@ -110,7 +110,7 @@ export async function createPluginConfigVM( } } } - + // run the plugin setup script, if present const __setupPlugin = async () => __callWithMeta('setupPlugin'); ` From 986c37db2a4430059c17923c92445e22e0f363bb Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 11:20:50 +0100 Subject: [PATCH 130/218] Start separating Postgres- and ClickHouse-based suites --- .github/workflows/ci.yml | 8 ++++---- package.json | 4 +++- tests/{ => postgres}/vm.test.ts | 0 3 files changed, 7 insertions(+), 5 deletions(-) rename tests/{ => postgres}/vm.test.ts (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7c494c4..d08e77e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Lint with ESLint run: yarn lint - tests: + tests-postgres: name: Tests / Postgres + Redis runs-on: ubuntu-20.04 @@ -92,7 +92,7 @@ jobs: # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' - run: cd plugin-server && yarn test + run: cd plugin-server && yarn test:postgres tests-clickhouse: name: Tests / ClickHouse + Kafka @@ -182,9 +182,9 @@ jobs: # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' - run: cd plugin-server && yarn test + run: cd plugin-server && yarn test:clickhouse - benchmarks: + benchmarks-postgres: name: Benchmarks / Postgres + Redis runs-on: ubuntu-20.04 diff --git a/package.json b/package.json index 93fdb684..eb764d87 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,9 @@ "types": "dist/src/index.d.ts", "main": "dist/src/index.js", "scripts": { - "test": "jest --runInBand tests/*.test.ts", + "test": "jest --runInBand 'tests/*.test.ts'", + "test:postgres": "yarn test --testPathIgnorePatterns '**/clickhouse/**/*'", + "test:clickhouse": "yarn test --testPathIgnorePatterns '**/postgres/**/*'", "benchmark": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/", "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", diff --git a/tests/vm.test.ts b/tests/postgres/vm.test.ts similarity index 100% rename from tests/vm.test.ts rename to tests/postgres/vm.test.ts From 4df3b3c8fc20014af2b9428696ff4b8a31045f7d Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 11:28:36 +0100 Subject: [PATCH 131/218] Fix regexes possibly --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index eb764d87..d60cfb75 100644 --- a/package.json +++ b/package.json @@ -5,9 +5,9 @@ "types": "dist/src/index.d.ts", "main": "dist/src/index.js", "scripts": { - "test": "jest --runInBand 'tests/*.test.ts'", - "test:postgres": "yarn test --testPathIgnorePatterns '**/clickhouse/**/*'", - "test:clickhouse": "yarn test --testPathIgnorePatterns '**/postgres/**/*'", + "test": "jest --runInBand tests/*.test.ts", + "test:postgres": "yarn test --testPathIgnorePatterns **/clickhouse/**/*", + "test:clickhouse": "yarn test --testPathIgnorePatterns **/postgres/**/*", "benchmark": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/", "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", From 2a260fb25f1e6f53d73039d8b7853561b2f3a315 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 11:47:24 +0100 Subject: [PATCH 132/218] Fix regexes actually possibly --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index d60cfb75..b9da191d 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "main": "dist/src/index.js", "scripts": { "test": "jest --runInBand tests/*.test.ts", - "test:postgres": "yarn test --testPathIgnorePatterns **/clickhouse/**/*", - "test:clickhouse": "yarn test --testPathIgnorePatterns **/postgres/**/*", + "test:postgres": "yarn test --testPathIgnorePatterns '.*/clickhouse'", + "test:clickhouse": "yarn test --testPathIgnorePatterns '.*/postgres'", "benchmark": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/", "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", From e0000f33fad938707e1b679bb885b0b18d6c376c Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 12:05:41 +0100 Subject: [PATCH 133/218] Improve fatal error handling --- src/ingestion/kafka-queue.ts | 3 ++- src/utils.ts | 13 +++++++++++++ src/worker/queue.ts | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 5fc3dd25..12ed969b 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -5,6 +5,7 @@ import { KAFKA_EVENTS_INGESTION_HANDOFF } from './topics' import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' import { parseRawEventMessage } from './utils' +import { killGracefully } from 'utils' export type BatchCallback = (messages: Message[]) => Promise @@ -131,7 +132,7 @@ export class KafkaQueue implements Queue { status.error('โš ๏ธ', `Kafka consumer group ${groupId} crashed!`) console.error(error) Sentry.captureException(error) - process.kill(process.pid, 'SIGINT') + killGracefully() }) consumer.on(CONNECT, () => { status.info('โœ…', 'Kafka consumer connected!') diff --git a/src/utils.ts b/src/utils.ts index 25aa4431..74e60ff5 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -5,6 +5,7 @@ import * as zlib from 'zlib' import { LogLevel } from './types' import { randomBytes } from 'crypto' import { DateTime } from 'luxon' +import { status } from './status' /** * @param binary Buffer @@ -280,3 +281,15 @@ export function delay(ms: number): Promise { setTimeout(resolve, ms) }) } + +/** Time until autoexit (due to error) gives up on graceful exit and kills the process right away. */ +const GRACEFUL_EXIT_PERIOD_SECONDS = 5 + +export function killGracefully(): void { + status.error('โฒ', 'Shutting plugin server down gracefully with SIGTERM...') + process.kill(process.pid, 'SIGTERM') + setTimeout(() => { + status.error('โฒ', `Plugin server still running after ${GRACEFUL_EXIT_PERIOD_SECONDS} s, killing it forcefully!`) + process.exit(1) + }, GRACEFUL_EXIT_PERIOD_SECONDS * 1000) +} diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 377520c5..74d1c95f 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -6,6 +6,7 @@ import Client from '../celery/client' import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' import { status } from '../status' +import { killGracefully } from 'utils' export async function startQueue( server: PluginsServer, @@ -17,8 +18,7 @@ export async function startQueue( return await relevantStartQueue(server, processEvent, processEventBatch) } catch (error) { status.error('๐Ÿ’ฅ', `Failed to start Kafka queue:\n${error}`) - process.kill(process.pid, 'SIGINT') - process.exit(1) + killGracefully() } } From 88009670ae92677dd7189f53ab9777590bbb68a6 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 12:12:35 +0100 Subject: [PATCH 134/218] Fix type --- src/utils.ts | 24 ++++++++++++------------ src/worker/queue.ts | 3 +-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 74e60ff5..b0d7578f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -7,6 +7,18 @@ import { randomBytes } from 'crypto' import { DateTime } from 'luxon' import { status } from './status' +/** Time until autoexit (due to error) gives up on graceful exit and kills the process right away. */ +const GRACEFUL_EXIT_PERIOD_SECONDS = 5 + +export function killGracefully(): void { + status.error('โฒ', 'Shutting plugin server down gracefully with SIGTERM...') + process.kill(process.pid, 'SIGTERM') + setTimeout(() => { + status.error('โฒ', `Plugin server still running after ${GRACEFUL_EXIT_PERIOD_SECONDS} s, killing it forcefully!`) + process.exit(1) + }, GRACEFUL_EXIT_PERIOD_SECONDS * 1000) +} + /** * @param binary Buffer * returns readableInstanceStream Readable @@ -281,15 +293,3 @@ export function delay(ms: number): Promise { setTimeout(resolve, ms) }) } - -/** Time until autoexit (due to error) gives up on graceful exit and kills the process right away. */ -const GRACEFUL_EXIT_PERIOD_SECONDS = 5 - -export function killGracefully(): void { - status.error('โฒ', 'Shutting plugin server down gracefully with SIGTERM...') - process.kill(process.pid, 'SIGTERM') - setTimeout(() => { - status.error('โฒ', `Plugin server still running after ${GRACEFUL_EXIT_PERIOD_SECONDS} s, killing it forcefully!`) - process.exit(1) - }, GRACEFUL_EXIT_PERIOD_SECONDS * 1000) -} diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 74d1c95f..29e42b05 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -6,7 +6,6 @@ import Client from '../celery/client' import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' import { status } from '../status' -import { killGracefully } from 'utils' export async function startQueue( server: PluginsServer, @@ -18,7 +17,7 @@ export async function startQueue( return await relevantStartQueue(server, processEvent, processEventBatch) } catch (error) { status.error('๐Ÿ’ฅ', `Failed to start Kafka queue:\n${error}`) - killGracefully() + process.exit(1) } } From ffe1977c68a523413d8fc0e692eaf7db277343ef Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 12:17:53 +0100 Subject: [PATCH 135/218] Fix import --- src/ingestion/kafka-queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 12ed969b..63ed0358 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -5,7 +5,7 @@ import { KAFKA_EVENTS_INGESTION_HANDOFF } from './topics' import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' import { parseRawEventMessage } from './utils' -import { killGracefully } from 'utils' +import { killGracefully } from '../utils' export type BatchCallback = (messages: Message[]) => Promise From bc832b7efa8175a40b0c7770a207d0008ef55408 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 12:34:02 +0100 Subject: [PATCH 136/218] Debug Kafka queue crash --- src/status.ts | 16 +++++++--------- src/worker/queue.ts | 4 ++-- tests/sql.test.ts | 2 ++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/status.ts b/src/status.ts index 0851f5ca..6af02cc9 100644 --- a/src/status.ts +++ b/src/status.ts @@ -1,23 +1,21 @@ import { threadId } from 'worker_threads' -export type StatusMethod = ((emoji: string, message: string) => void) | ((message: string) => void) +export type StatusMethod = (icon: string, ...message: string[]) => void export interface Status { info: StatusMethod error: StatusMethod } -function transform(emojiOrMessage: string, message?: string): string { - return `[${threadId ? threadId.toString().padStart(4, '_') : 'MAIN'}] ${[emojiOrMessage, message] - .filter(Boolean) - .join(' ')}` +function getPrefix(): string { + return `[${threadId ? threadId.toString().padStart(4, '_') : 'MAIN'}]` } export const status: Status = { - info(emojiOrMessage: string, message?: string) { - console.info(transform(emojiOrMessage, message)) + info(icon: string, ...message: string[]) { + console.info(getPrefix(), icon, ...message.filter(Boolean)) }, - error(emojiOrMessage: string, message?: string) { - console.error(transform(emojiOrMessage, message)) + error(icon: string, ...message: string[]) { + console.error(getPrefix(), icon, ...message.filter(Boolean)) }, } diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 29e42b05..d8fbe202 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -16,8 +16,8 @@ export async function startQueue( try { return await relevantStartQueue(server, processEvent, processEventBatch) } catch (error) { - status.error('๐Ÿ’ฅ', `Failed to start Kafka queue:\n${error}`) - process.exit(1) + status.error('๐Ÿ’ฅ', 'Failed to start Kafka queue:\n', error) + throw error } } diff --git a/tests/sql.test.ts b/tests/sql.test.ts index c7afa9f8..050182c1 100644 --- a/tests/sql.test.ts +++ b/tests/sql.test.ts @@ -2,6 +2,7 @@ import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } import { PluginConfig, PluginError, PluginsServer } from '../src/types' import { createServer } from '../src/server' import { resetTestDatabase } from './helpers/sql' +import { organizationId } from './helpers/plugins' let server: PluginsServer let closeServer: () => Promise @@ -83,6 +84,7 @@ test('getPluginRows', async () => { from_json: false, from_web: false, id: 60, + organization_id: organizationId, name: 'test-maxmind-plugin', plugin_type: 'custom', source: null, From 5164d18a32b2e4b1d08691221c5761be9535e5f3 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 12:58:25 +0100 Subject: [PATCH 137/218] Debug consumer --- src/ingestion/kafka-queue.ts | 1 + src/server.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 63ed0358..9dfb1967 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -124,6 +124,7 @@ export class KafkaQueue implements Queue { groupId: 'clickhouse-ingestion', readUncommitted: false, }) + console.log('Consumer:', consumer) const { GROUP_JOIN, CRASH, CONNECT, DISCONNECT } = consumer.events consumer.on(GROUP_JOIN, ({ payload: { groupId } }) => { status.info('โœ…', `Kafka consumer joined group ${groupId}!`) diff --git a/src/server.ts b/src/server.ts index e62a6459..61267823 100644 --- a/src/server.ts +++ b/src/server.ts @@ -184,7 +184,9 @@ export async function startPluginsServer( pingJob && schedule.cancelJob(pingJob) statsJob && schedule.cancelJob(statsJob) await stopSchedule?.() - await stopPiscina(piscina!) + if (piscina) { + await stopPiscina(piscina) + } await closeServer() // wait an extra second for any misc async task to finish From b0e5ba518d7f2c786a20c0b9d23800b572c222ad Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 13:11:55 +0100 Subject: [PATCH 138/218] Fix organizationId --- tests/helpers/plugins.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/plugins.ts b/tests/helpers/plugins.ts index 0b0e33a5..b724fd83 100644 --- a/tests/helpers/plugins.ts +++ b/tests/helpers/plugins.ts @@ -4,7 +4,7 @@ import path from 'path' import os from 'os' import AdmZip from 'adm-zip' -export const organizationId = 'CA30F2EC-E9A4-4001-BF27-3EF194086068' +export const organizationId = 'ca30f2ec-e9a4-4001-bf27-3ef194086068' export const plugin60: Plugin & { organization_id: string } = { id: 60, From 77c0c41eb1553977518e95dba89d1656bfb29c9f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 13:12:07 +0100 Subject: [PATCH 139/218] Make some more suites Postgres-specific --- tests/{ => postgres}/plugins.test.ts | 0 tests/{ => postgres}/queue.test.ts | 0 tests/{ => postgres}/worker.test.ts | 16 ++++++++-------- 3 files changed, 8 insertions(+), 8 deletions(-) rename tests/{ => postgres}/plugins.test.ts (100%) rename tests/{ => postgres}/queue.test.ts (100%) rename tests/{ => postgres}/worker.test.ts (95%) diff --git a/tests/plugins.test.ts b/tests/postgres/plugins.test.ts similarity index 100% rename from tests/plugins.test.ts rename to tests/postgres/plugins.test.ts diff --git a/tests/queue.test.ts b/tests/postgres/queue.test.ts similarity index 100% rename from tests/queue.test.ts rename to tests/postgres/queue.test.ts diff --git a/tests/worker.test.ts b/tests/postgres/worker.test.ts similarity index 95% rename from tests/worker.test.ts rename to tests/postgres/worker.test.ts index 898808ee..a58e4a7f 100644 --- a/tests/worker.test.ts +++ b/tests/postgres/worker.test.ts @@ -1,11 +1,11 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' -import { setupPiscina } from './helpers/worker' -import { delay } from '../src/utils' -import { startPluginsServer } from '../src/server' -import { LogLevel } from '../src/types' -import { makePiscina } from '../src/worker/piscina' -import Client from '../src/celery/client' -import { resetTestDatabase } from './helpers/sql' +import { setupPiscina } from '../helpers/worker' +import { delay } from '../../src/utils' +import { startPluginsServer } from '../../src/server' +import { LogLevel } from '../../src/types' +import { makePiscina } from '../../src/worker/piscina' +import Client from '../../src/celery/client' +import { resetTestDatabase } from '../helpers/sql' jest.mock('../src/sql') jest.setTimeout(600000) // 600 sec timeout @@ -31,7 +31,7 @@ test('piscina worker test', async () => { } async function runEveryDay (meta) { return 4 - } + } ` await resetTestDatabase(testCode) const piscina = setupPiscina(workerThreads, 10) From 76ea34e0cc5777a7b1a1ad7904bed751542f0c19 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 21 Jan 2021 16:25:57 +0100 Subject: [PATCH 140/218] Check out master branch of main repo of Django server in CI --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d08e77e5..96b84707 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,6 @@ jobs: uses: actions/checkout@v2 with: repository: 'PostHog/posthog' - ref: 'fix-ch-compose' # TODO: remove this line to use master path: 'posthog/' - name: Check out plugin server @@ -145,7 +144,6 @@ jobs: uses: actions/checkout@v2 with: repository: 'PostHog/posthog' - ref: 'fix-ch-compose' # TODO: remove this line to use master path: 'posthog/' - name: Check out plugin server @@ -215,7 +213,6 @@ jobs: uses: actions/checkout@v2 with: repository: 'PostHog/posthog' - ref: 'fix-ch-compose' # TODO: remove this line to use master path: 'posthog/' - name: Check out plugin server From 0abb0eabdb9a21a96b4959bf473f6b54d03b2857 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 22 Jan 2021 01:20:56 +0100 Subject: [PATCH 141/218] Start schedule before queue --- src/server.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server.ts b/src/server.ts index fcd53993..c4e917d6 100644 --- a/src/server.ts +++ b/src/server.ts @@ -228,6 +228,7 @@ export async function startPluginsServer( fastifyInstance = await startFastifyInstance(server) } + stopSchedule = await startSchedule(server, piscina) queue = await startQueue(server, processEvent, processEventBatch) piscina.on('drain', () => { queue?.resume() @@ -264,8 +265,6 @@ export async function startPluginsServer( } }) - stopSchedule = await startSchedule(server, piscina) - status.info('๐Ÿš€', 'All systems go.') } catch (error) { Sentry.captureException(error) From 8d0187e1a6753eff391bfe53afe5e0095f51ce7f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 22 Jan 2021 01:21:04 +0100 Subject: [PATCH 142/218] Clean up logging --- src/ingestion/kafka-queue.ts | 1 - src/services/schedule.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 764b24fc..6eac4154 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -135,7 +135,6 @@ export class KafkaQueue implements Queue { groupId: 'clickhouse-ingestion', readUncommitted: false, }) - console.log('Consumer:', consumer) const { GROUP_JOIN, CRASH, CONNECT, DISCONNECT } = consumer.events consumer.on(GROUP_JOIN, ({ payload: { groupId } }) => { status.info('โœ…', `Kafka consumer joined group ${groupId}!`) diff --git a/src/services/schedule.ts b/src/services/schedule.ts index 9a2db4f9..219ca177 100644 --- a/src/services/schedule.ts +++ b/src/services/schedule.ts @@ -13,7 +13,7 @@ export async function startSchedule( piscina: Piscina, onLock?: () => void ): Promise<() => Promise> { - status.info('โฐ', 'Starting scheduling service') + status.info('โฐ', 'Starting scheduling service...') let stopped = false let weHaveTheLock = false From 4acac58299696b5f52f6cb00a3fccd9b049798d7 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 22 Jan 2021 14:00:25 +0100 Subject: [PATCH 143/218] Update Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e754e9a9..e33f6bdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,10 @@ FROM node:14 WORKDIR /code COPY package.json yarn.lock .eslintrc.js .prettierrc ./ +COPY src/idl/ src/idl/ RUN yarn install --frozen-lockfile COPY ./ ./ -RUN yarn compile +RUN yarn compile:typescript CMD [ "node", "dist/src/index.js" ] From 6ada689eb278f88abe4dfd7f0f445115bd8bb402 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 25 Jan 2021 11:14:05 +0100 Subject: [PATCH 144/218] Refactor Postgres processing to DB class --- src/db.ts | 138 ++++++++++++++++++++++++++ src/ingestion/process-event.ts | 174 +++++---------------------------- src/server.ts | 23 +++-- src/types.ts | 8 +- src/utils.ts | 7 +- 5 files changed, 183 insertions(+), 167 deletions(-) create mode 100644 src/db.ts diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 00000000..0fac596f --- /dev/null +++ b/src/db.ts @@ -0,0 +1,138 @@ +import { Properties } from '@posthog/plugin-scaffold' +import { ClickHouse } from 'clickhouse' +import { Producer } from 'kafkajs' +import { DateTime } from 'luxon' +import { Pool, PoolConfig } from 'pg' +import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' +import { unparsePersonPartial } from './ingestion/utils' +import { Person, PersonDistinctId, RawPerson } from './types' +import { castTimestampOrNow } from './utils' + +export class DB extends Pool { + /** Kafka producer used for syncing Postgres and ClickHouse person data. */ + kafkaProducer?: Producer + /** ClickHouse used for syncing Postgres and ClickHouse person data. */ + clickhouse?: ClickHouse + + constructor(config: PoolConfig, kafkaProducer?: Producer, clickhouse?: ClickHouse) { + super(config) + this.kafkaProducer = kafkaProducer + this.clickhouse = clickhouse + } + + public async fetchPerson(teamId: number, distinctId: string): Promise { + const selectResult = await this.query( + `SELECT + posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, + posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, + posthog_persondistinctid.team_id AS persondistinctid__team_id, + posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id + FROM posthog_person + JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) + WHERE + posthog_person.team_id = $1 + AND posthog_persondistinctid.team_id = $1 + AND posthog_persondistinctid.distinct_id = $2`, + [teamId, distinctId] + ) + const rawPerson: RawPerson = selectResult.rows[0] + return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } + } + + public async createPerson( + createdAt: DateTime, + properties: Properties, + teamId: number, + isUserId: number | null, + isIdentified: boolean, + uuid: string + ): Promise { + const insertResult = await this.query( + 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', + [createdAt.toISO(), JSON.stringify(properties), teamId, isUserId, isIdentified, uuid] + ) + const personCreated = insertResult.rows[0] as Person + if (this.kafkaProducer) { + const data = { + created_at: castTimestampOrNow(createdAt), + properties: JSON.stringify(properties), + team_id: teamId, + is_identified: isIdentified, + id: uuid, + } + await this.kafkaProducer.send({ + topic: KAFKA_PERSON, + messages: [{ value: Buffer.from(JSON.stringify(data)) }], + }) + } + return personCreated + } + + public async updatePerson(person: Person, update: Partial): Promise { + const updatedPerson: Person = { ...person, ...update } + await this.query( + `UPDATE posthog_person SET ${Object.keys(update).map( + (field, index) => field + ' = $' + (index + 1) + )} WHERE id = $${Object.values(update).length + 1}`, + [...Object.values(unparsePersonPartial(update)), person.id] + ) + if (this.kafkaProducer) { + const data = { + created_at: castTimestampOrNow(updatedPerson.created_at), + properties: JSON.stringify(updatedPerson.properties), + team_id: updatedPerson.team_id, + is_identified: updatedPerson.is_identified, + id: updatedPerson.uuid.toString(), + } + await this.kafkaProducer.send({ + topic: KAFKA_PERSON, + messages: [{ value: Buffer.from(JSON.stringify(data)) }], + }) + } + return updatedPerson + } + + public async deletePerson(personId: number): Promise { + await this.query('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) + await this.query('DELETE FROM posthog_person WHERE id = $1', [personId]) + if (this.clickhouse) { + await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() + await this.clickhouse + .query(`ALTER TABLE person_distinct_id DELETE WHERE person_id = ${personId}`) + .toPromise() + } + } + + public async addDistinctId(person: Person, distinctId: string): Promise { + const insertResult = await this.query( + 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', + [distinctId, person.id, person.team_id] + ) + const personDistinctIdCreated = insertResult.rows[0] as PersonDistinctId + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_PERSON_UNIQUE_ID, + messages: [{ value: Buffer.from(JSON.stringify(personDistinctIdCreated)) }], + }) + } + } + + public async updateDistinctId( + personDistinctId: PersonDistinctId, + update: Partial + ): Promise { + const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } + await this.query( + `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( + (field, index) => field + ' = $' + (index + 1) + )} WHERE id = $${Object.values(update).length + 1}`, + [...Object.values(update), personDistinctId.id] + ) + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_PERSON_UNIQUE_ID, + messages: [{ value: Buffer.from(JSON.stringify(updatedPersonDistinctId)) }], + }) + } + } +} diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 085fcacf..42b59796 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,27 +1,17 @@ -import { PluginEvent } from '@posthog/plugin-scaffold' +import { PluginEvent, Properties } from '@posthog/plugin-scaffold' import { DateTime, Duration } from 'luxon' -import { - PluginsServer, - EventData, - Properties, - Element, - Team, - Person, - PersonDistinctId, - CohortPeople, - RawPerson, -} from '../types' +import { PluginsServer, EventData, Element, Team, Person, PersonDistinctId, CohortPeople } from '../types' import { castTimestampOrNow, UUIDT } from '../utils' import { Event as EventProto } from '../idl/protos' -import { Pool } from 'pg' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { sanitizeEventName, elementsToString, unparsePersonPartial } from './utils' import { ClickHouse } from 'clickhouse' +import { DB } from '../db' export class EventsProcessor { pluginsServer: PluginsServer - db: Pool + db: DB clickhouse: ClickHouse kafkaProducer: Producer @@ -129,10 +119,10 @@ export class EventsProcessor { } private async setIsIdentified(teamId: number, distinctId: string, isIdentified = true): Promise { - let personFound = await this.fetchPerson(teamId, distinctId) + let personFound = await this.db.fetchPerson(teamId, distinctId) if (!personFound) { try { - const personCreated = await this.createPerson( + const personCreated = await this.db.createPerson( DateTime.utc(), {}, teamId, @@ -140,11 +130,11 @@ export class EventsProcessor { true, new UUIDT().toString() ) - this.addDistinctId(personCreated, distinctId) + this.db.addDistinctId(personCreated, distinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person - personFound = await this.fetchPerson(teamId, distinctId) + personFound = await this.db.fetchPerson(teamId, distinctId) } } if (personFound && !personFound.is_identified) { @@ -161,10 +151,10 @@ export class EventsProcessor { properties: Properties, setOnce = false ): Promise { - let personFound = await this.fetchPerson(teamId, distinctId) + let personFound = await this.db.fetchPerson(teamId, distinctId) if (!personFound) { try { - const personCreated = await this.createPerson( + const personCreated = await this.db.createPerson( DateTime.utc(), properties, teamId, @@ -172,11 +162,11 @@ export class EventsProcessor { false, new UUIDT().toString() ) - await this.addDistinctId(personCreated, distinctId) + await this.db.addDistinctId(personCreated, distinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person - personFound = await this.fetchPerson(teamId, distinctId) + personFound = await this.db.fetchPerson(teamId, distinctId) } } this.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ @@ -193,12 +183,12 @@ export class EventsProcessor { teamId: number, retryIfFailed = true ): Promise { - const oldPerson = await this.fetchPerson(teamId, previousDistinctId) - const newPerson = await this.fetchPerson(teamId, distinctId) + const oldPerson = await this.db.fetchPerson(teamId, previousDistinctId) + const newPerson = await this.db.fetchPerson(teamId, distinctId) if (oldPerson && !newPerson) { try { - this.addDistinctId(oldPerson, distinctId) + this.db.addDistinctId(oldPerson, distinctId) // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error @@ -212,7 +202,7 @@ export class EventsProcessor { if (!oldPerson && newPerson) { try { - this.addDistinctId(newPerson, previousDistinctId) + this.db.addDistinctId(newPerson, previousDistinctId) // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error @@ -226,7 +216,7 @@ export class EventsProcessor { if (!oldPerson && !newPerson) { try { - const personCreated = await this.createPerson( + const personCreated = await this.db.createPerson( DateTime.utc(), {}, teamId, @@ -234,8 +224,8 @@ export class EventsProcessor { false, new UUIDT().toString() ) - this.addDistinctId(personCreated, distinctId) - this.addDistinctId(personCreated, previousDistinctId) + this.db.addDistinctId(personCreated, distinctId) + this.db.addDistinctId(personCreated, previousDistinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person @@ -264,7 +254,7 @@ export class EventsProcessor { } } - await this.updatePerson(mergeInto, { created_at: first_seen }) + await this.db.updatePerson(mergeInto, { created_at: first_seen }) // merge the distinct_ids for (const other_person of peopleToMerge) { @@ -275,7 +265,7 @@ export class EventsProcessor { ]) ).rows for (const person_distinct_id of other_person_distinct_ids) { - await this.updateDistinctId(person_distinct_id, { person_id: mergeInto.id }) + await this.db.updateDistinctId(person_distinct_id, { person_id: mergeInto.id }) } const other_person_cohort_ids: CohortPeople[] = ( @@ -288,7 +278,7 @@ export class EventsProcessor { ]) } - await this.deletePerson(other_person.id) + await this.db.deletePerson(other_person.id) } } @@ -339,7 +329,7 @@ export class EventsProcessor { if (!pdiCount) { // Catch race condition where in between getting and creating, another request already created this user try { - const personCreated: Person = await this.createPerson( + const personCreated: Person = await this.db.createPerson( sentAt || DateTime.utc(), {}, teamId, @@ -347,7 +337,7 @@ export class EventsProcessor { false, personUuid.toString() ) - await this.addDistinctId(personCreated, distinctId) + await this.db.addDistinctId(personCreated, distinctId) } catch {} } @@ -402,122 +392,6 @@ export class EventsProcessor { } } - private async fetchPerson(teamId: number, distinctId: string): Promise { - const selectResult = await this.db.query( - `SELECT - posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, - posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, - posthog_persondistinctid.team_id AS persondistinctid__team_id, - posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id - FROM posthog_person - JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) - WHERE - posthog_person.team_id = $1 - AND posthog_persondistinctid.team_id = $1 - AND posthog_persondistinctid.distinct_id = $2`, - [teamId, distinctId] - ) - const rawPerson: RawPerson = selectResult.rows[0] - return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } - } - - private async createPerson( - createdAt: DateTime, - properties: Properties, - teamId: number, - isUserId: number | null, - isIdentified: boolean, - uuid: string - ): Promise { - const insertResult = await this.db.query( - 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', - [createdAt.toISO(), JSON.stringify(properties), teamId, isUserId, isIdentified, uuid] - ) - const personCreated = insertResult.rows[0] as Person - if (this.pluginsServer.KAFKA_ENABLED) { - const data = { - created_at: castTimestampOrNow(createdAt), - properties: JSON.stringify(properties), - team_id: teamId, - is_identified: isIdentified, - id: uuid, - } - await this.kafkaProducer.send({ - topic: KAFKA_PERSON, - messages: [{ value: Buffer.from(JSON.stringify(data)) }], - }) - } - return personCreated - } - - private async updatePerson(person: Person, update: Partial): Promise { - const updatedPerson: Person = { ...person, ...update } - await this.db.query( - `UPDATE posthog_person SET ${Object.keys(update).map( - (field, index) => field + ' = $' + (index + 1) - )} WHERE id = $${Object.values(update).length + 1}`, - [...Object.values(unparsePersonPartial(update)), person.id] - ) - if (this.pluginsServer.KAFKA_ENABLED) { - const data = { - created_at: castTimestampOrNow(updatedPerson.created_at), - properties: JSON.stringify(updatedPerson.properties), - team_id: updatedPerson.team_id, - is_identified: updatedPerson.is_identified, - id: updatedPerson.uuid.toString(), - } - await this.kafkaProducer.send({ - topic: KAFKA_PERSON, - messages: [{ value: Buffer.from(JSON.stringify(data)) }], - }) - } - return updatedPerson - } - - private async deletePerson(personId: number): Promise { - await this.db.query('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) - await this.db.query('DELETE FROM posthog_person WHERE id = $1', [personId]) - if (this.pluginsServer.KAFKA_ENABLED) { - await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() - await this.clickhouse - .query(`ALTER TABLE person_distinct_id DELETE WHERE person_id = ${personId}`) - .toPromise() - } - } - - private async addDistinctId(person: Person, distinctId: string): Promise { - const insertResult = await this.db.query( - 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', - [distinctId, person.id, person.team_id] - ) - const personDistinctIdCreated = insertResult.rows[0] as PersonDistinctId - if (this.pluginsServer.KAFKA_ENABLED) { - await this.kafkaProducer.send({ - topic: KAFKA_PERSON_UNIQUE_ID, - messages: [{ value: Buffer.from(JSON.stringify(personDistinctIdCreated)) }], - }) - } - } - - private async updateDistinctId( - personDistinctId: PersonDistinctId, - update: Partial - ): Promise { - const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } - await this.db.query( - `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( - (field, index) => field + ' = $' + (index + 1) - )} WHERE id = $${Object.values(update).length + 1}`, - [...Object.values(update), personDistinctId.id] - ) - if (this.pluginsServer.KAFKA_ENABLED) { - await this.kafkaProducer.send({ - topic: KAFKA_PERSON_UNIQUE_ID, - messages: [{ value: Buffer.from(JSON.stringify(updatedPersonDistinctId)) }], - }) - } - } - private async createEvent( uuid: string, event: string, diff --git a/src/server.ts b/src/server.ts index 7517b145..0fa068c3 100644 --- a/src/server.ts +++ b/src/server.ts @@ -18,6 +18,7 @@ import { EventsProcessor } from './ingestion/process-event' import { status } from './status' import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' +import { DB } from './db' export async function createServer( config: Partial = {}, @@ -41,15 +42,6 @@ export async function createServer( }) await redis.info() - const db = new Pool({ - connectionString: serverConfig.DATABASE_URL, - ssl: process.env.DEPLOYMENT?.startsWith('Heroku') - ? { - rejectUnauthorized: false, - } - : undefined, - }) - let kafkaSsl: ConnectionOptions | undefined if ( serverConfig.KAFKA_CLIENT_CERT_B64 && @@ -96,6 +88,19 @@ export async function createServer( kafkaProducer = kafka.producer() } + const db = new DB( + { + connectionString: serverConfig.DATABASE_URL, + ssl: process.env.DEPLOYMENT?.startsWith('Heroku') + ? { + rejectUnauthorized: false, + } + : undefined, + }, + kafkaProducer, + clickhouse + ) + let statsd: StatsD | undefined if (serverConfig.STATSD_HOST) { statsd = new StatsD({ diff --git a/src/types.ts b/src/types.ts index 303f06ff..ada03273 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,12 +1,13 @@ import { Pool } from 'pg' import { Redis } from 'ioredis' import { Kafka, Producer } from 'kafkajs' -import { PluginEvent, PluginAttachment, PluginConfigSchema } from '@posthog/plugin-scaffold' +import { PluginEvent, PluginAttachment, PluginConfigSchema, Properties } from '@posthog/plugin-scaffold' import { VM } from 'vm2' import { DateTime } from 'luxon' import { StatsD } from 'hot-shots' import { EventsProcessor } from 'ingestion/process-event' import { ClickHouse } from 'clickhouse' +import { DB } from './db' export enum LogLevel { Debug = 'debug', @@ -50,7 +51,7 @@ export interface PluginsServerConfig extends Record { export interface PluginsServer extends PluginsServerConfig { // active connections to Postgres, Redis, ClickHouse, Kafka, StatsD - db: Pool + db: DB redis: Redis clickhouse?: ClickHouse kafka?: Kafka @@ -201,9 +202,6 @@ export interface EventData extends PluginEvent { offset?: number } -/** Any kind of properties field. */ -export type Properties = Record - /** Usable Team model. */ export interface Team { id: number diff --git a/src/utils.ts b/src/utils.ts index b0d7578f..42776b90 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -279,13 +279,14 @@ export class UUIDT extends UUID { } } +/** Format timestamp for ClickHouse. */ export function castTimestampOrNow(timestamp?: DateTime | string | null): string { if (!timestamp) { timestamp = DateTime.utc() + } else if (typeof timestamp === 'string') { + timestamp = DateTime.fromISO(timestamp) } - // ClickHouse-specific formatting - timestamp = typeof timestamp === 'string' ? DateTime.fromISO(timestamp) : timestamp.toUTC() - return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') + return timestamp.toUTC().toFormat('yyyy-MM-dd HH:mm:ss.u') } export function delay(ms: number): Promise { From 13f5e3160b01c13830f4bf35985014e57e22eba4 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 25 Jan 2021 11:16:59 +0100 Subject: [PATCH 145/218] Use more of DB methods --- src/ingestion/process-event.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 42b59796..954448f0 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -138,10 +138,7 @@ export class EventsProcessor { } } if (personFound && !personFound.is_identified) { - await this.db.query('UPDATE posthog_person SET is_identified = $1 WHERE id = $2', [ - isIdentified, - personFound.id, - ]) + await this.db.updatePerson(personFound, { is_identified: isIdentified }) } } @@ -169,12 +166,10 @@ export class EventsProcessor { personFound = await this.db.fetchPerson(teamId, distinctId) } } - this.db.query('UPDATE posthog_person SET properties = $1 WHERE id = $2', [ - JSON.stringify( - setOnce ? { ...properties, ...personFound!.properties } : { ...personFound!.properties, ...properties } - ), - personFound!.id, - ]) + this.db.updatePerson( + personFound!, + setOnce ? { ...properties, ...personFound!.properties } : { ...personFound!.properties, ...properties } + ) } private async alias( From 07fb5e13470a799c84caaee5c0354349163ef0d8 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 25 Jan 2021 11:58:56 +0100 Subject: [PATCH 146/218] Quit Redis --- src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.ts b/src/server.ts index 0fa068c3..6c4fcc24 100644 --- a/src/server.ts +++ b/src/server.ts @@ -188,6 +188,7 @@ export async function startPluginsServer( await stopFastifyInstance(fastifyInstance!) } await queue?.stop() + await pubSub?.quit() await kafkaProducer?.disconnect() pubSub?.disconnect() pingJob && schedule.cancelJob(pingJob) From 19c8f7eb1a411647359164ec62f52b48608dc8fe Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 25 Jan 2021 11:59:06 +0100 Subject: [PATCH 147/218] Update test database name --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 28d5ca8d..c8f9036d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,7 +8,7 @@ export function getDefaultConfig(): PluginsServerConfig { return { CELERY_DEFAULT_QUEUE: 'celery', - DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/posthog_test' : 'postgres://localhost:5432/posthog', + DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/test_posthog' : 'postgres://localhost:5432/posthog', CLICKHOUSE_HOST: 'localhost', CLICKHOUSE_DATABASE: 'default', CLICKHOUSE_USERNAME: 'default', From cc094c4449c41f1890490d2fff649dc1338e79f0 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 25 Jan 2021 12:17:02 +0100 Subject: [PATCH 148/218] Debug consumer --- src/ingestion/kafka-queue.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 6eac4154..b91712bc 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -23,8 +23,10 @@ export class KafkaQueue implements Queue { saveEvent: (event: PluginEvent) => Promise ) { this.pluginsServer = pluginsServer + console.log('Kafka:', pluginsServer.kafka) this.kafka = pluginsServer.kafka! this.consumer = KafkaQueue.buildConsumer(this.kafka) + console.log('Consumer constructed:', this.consumer) this.wasConsumerRan = false this.processEventBatch = processEventBatch this.saveEvent = saveEvent @@ -135,6 +137,7 @@ export class KafkaQueue implements Queue { groupId: 'clickhouse-ingestion', readUncommitted: false, }) + console.log('Consumer:', consumer) const { GROUP_JOIN, CRASH, CONNECT, DISCONNECT } = consumer.events consumer.on(GROUP_JOIN, ({ payload: { groupId } }) => { status.info('โœ…', `Kafka consumer joined group ${groupId}!`) From c139cf106e4dca426cefec258250e711b987f4b7 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 25 Jan 2021 12:28:44 +0100 Subject: [PATCH 149/218] Don't mock KafkaJS! --- jest.config.js | 2 +- jest.setup.kafka-mock.js | 1 - src/ingestion/kafka-queue.ts | 3 --- 3 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 jest.setup.kafka-mock.js diff --git a/jest.config.js b/jest.config.js index c8474d90..c4967502 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,6 +3,6 @@ module.exports = { testEnvironment: 'node', clearMocks: true, coverageProvider: 'v8', - setupFilesAfterEnv: ['./jest.setup.kafka-mock.js', './jest.setup.fetch-mock.js'], + setupFilesAfterEnv: ['./jest.setup.fetch-mock.js'], testMatch: ['/tests/**/*.test.ts', '/benchmarks/**/*.benchmark.ts'], } diff --git a/jest.setup.kafka-mock.js b/jest.setup.kafka-mock.js deleted file mode 100644 index 67300f89..00000000 --- a/jest.setup.kafka-mock.js +++ /dev/null @@ -1 +0,0 @@ -jest.mock('kafkajs') diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index b91712bc..6eac4154 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -23,10 +23,8 @@ export class KafkaQueue implements Queue { saveEvent: (event: PluginEvent) => Promise ) { this.pluginsServer = pluginsServer - console.log('Kafka:', pluginsServer.kafka) this.kafka = pluginsServer.kafka! this.consumer = KafkaQueue.buildConsumer(this.kafka) - console.log('Consumer constructed:', this.consumer) this.wasConsumerRan = false this.processEventBatch = processEventBatch this.saveEvent = saveEvent @@ -137,7 +135,6 @@ export class KafkaQueue implements Queue { groupId: 'clickhouse-ingestion', readUncommitted: false, }) - console.log('Consumer:', consumer) const { GROUP_JOIN, CRASH, CONNECT, DISCONNECT } = consumer.events consumer.on(GROUP_JOIN, ({ payload: { groupId } }) => { status.info('โœ…', `Kafka consumer joined group ${groupId}!`) From 958f24cc5908139580e545ad52bfd621a094890e Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 01:32:24 +0100 Subject: [PATCH 150/218] Add new Kafka testing system incomplete --- src/config.ts | 2 +- tests/clickhouse/process-event.ts | 41 +++++++++++++++++ tests/helpers/clickhouse.ts | 24 ++++++++++ tests/helpers/kafka.ts | 74 +++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 tests/clickhouse/process-event.ts create mode 100644 tests/helpers/clickhouse.ts create mode 100644 tests/helpers/kafka.ts diff --git a/src/config.ts b/src/config.ts index c8f9036d..24e53d0a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,7 +10,7 @@ export function getDefaultConfig(): PluginsServerConfig { CELERY_DEFAULT_QUEUE: 'celery', DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/test_posthog' : 'postgres://localhost:5432/posthog', CLICKHOUSE_HOST: 'localhost', - CLICKHOUSE_DATABASE: 'default', + CLICKHOUSE_DATABASE: isTestEnv ? 'test_posthog' : 'default', CLICKHOUSE_USERNAME: 'default', CLICKHOUSE_PASSWORD: null, CLICKHOUSE_CA: null, diff --git a/tests/clickhouse/process-event.ts b/tests/clickhouse/process-event.ts new file mode 100644 index 00000000..ee606ae5 --- /dev/null +++ b/tests/clickhouse/process-event.ts @@ -0,0 +1,41 @@ +import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } from '../../src/sql' +import { PluginConfig, PluginError, PluginsServer } from '../../src/types' +import { createServer } from '../../src/server' +import { resetTestDatabase } from '../helpers/sql' +import { organizationId } from '../helpers/plugins' +import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' +import { Consumer, EachMessagePayload, Kafka, Producer } from 'kafkajs' +import { KafkaObserver } from '../helpers/kafka' + +let server: PluginsServer +let closeServer: () => Promise +kafkaObserver = new KafkaObserver() + +beforeEach(async () => { + ;[server, closeServer] = await createServer() + await resetTestDatabase(`const processEvent = event => event`) + await resetTestDatabaseClickhouse() + await kafkaObserver.start() +}) +afterEach(() => { + closeServer() +}) + +test('event is passed through', async () => { + const rows1 = await getPluginAttachmentRows(server) + expect(rows1).toEqual([ + { + content_type: 'application/octet-stream', + contents: Buffer.from([116, 101, 115, 116]), + file_name: 'test.txt', + file_size: 4, + id: 1, + key: 'maxmindMmdb', + plugin_config_id: 39, + team_id: 2, + }, + ]) + server.db.query("update posthog_team set plugins_opt_in='f'") + const rows2 = await getPluginAttachmentRows(server) + expect(rows2).toEqual([]) +}) diff --git a/tests/helpers/clickhouse.ts b/tests/helpers/clickhouse.ts new file mode 100644 index 00000000..936d5e21 --- /dev/null +++ b/tests/helpers/clickhouse.ts @@ -0,0 +1,24 @@ +import { defaultConfig } from '../../src/config' +import { ClickHouse } from 'clickhouse' + +export async function resetTestDatabaseClickhouse(): Promise { + const clickhouse = new ClickHouse({ + url: `http://$${defaultConfig.CLICKHOUSE_HOST}`, + port: 8123, + config: { + database: defaultConfig.CLICKHOUSE_DATABASE, + }, + }) + await clickhouse.query('TRUNCATE events').toPromise() + await clickhouse.query('TRUNCATE events_mv').toPromise() + await clickhouse.query('TRUNCATE kafka_events').toPromise() + await clickhouse.query('TRUNCATE kafka_person').toPromise() + await clickhouse.query('TRUNCATE kafka_person_distinct_id').toPromise() + await clickhouse.query('TRUNCATE kafka_session_recording_events').toPromise() + await clickhouse.query('TRUNCATE person').toPromise() + await clickhouse.query('TRUNCATE person_distinct_id').toPromise() + await clickhouse.query('TRUNCATE person_mv').toPromise() + await clickhouse.query('TRUNCATE person_static_cohort').toPromise() + await clickhouse.query('TRUNCATE session_recording_events').toPromise() + await clickhouse.query('TRUNCATE session_recording_events_mv').toPromise() +} diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts new file mode 100644 index 00000000..515748c7 --- /dev/null +++ b/tests/helpers/kafka.ts @@ -0,0 +1,74 @@ +import { EventEmitter } from 'events' +import { Kafka, Consumer, logLevel, EachMessagePayload, Producer } from 'kafkajs' +import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' +import { parseRawEventMessage } from '../../src/ingestion/utils' +import { EventMessage, RawEventMessage } from '../../src/types' +import { UUIDT } from '../../src/utils' + +export class KafkaObserver extends EventEmitter { + public kafka: Kafka + public producer: Producer + public consumer: Consumer + + private isStarted: boolean + + constructor() { + super() + this.kafka = new Kafka({ + clientId: `plugin-server-test-${new UUIDT()}`, + brokers: process.env.KAFKA_HOSTS!.split(','), + logLevel: logLevel.NOTHING, + }) + this.producer = this.kafka.producer() + this.consumer = this.kafka.consumer({ + groupId: 'clickhouse-ingestion', + readUncommitted: false, + }) + this.isStarted = false + } + + public async start(): Promise { + if (this.isStarted) {return} + this.isStarted = true + return await new Promise(async (resolve, reject) => { + await this.consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) + await this.consumer.run({ + eachMessage: async (payload) => { + this.emit('message', payload) + }, + }) + const { GROUP_JOIN, CRASH } = this.consumer.events + this.consumer.on(GROUP_JOIN, () => resolve()) + this.consumer.on(CRASH, ({ payload: { error } }) => reject(error)) + }) + } + + public async stop(): Promise { + this.removeAllListeners() + await this.consumer.stop() + await this.consumer.disconnect() + await this.producer.disconnect() + } + + public handOffMessa + + public async waitForProcessedMessages(numberOfMessages: number): Promise { + return await new Promise((resolve, reject) => { + const accumulator: EventMessage[] = [] + const timeoutSeconds = numberOfMessages * 2 // give every message 2 s on average to show up + setTimeout( + () => + reject(`Timed out waiting ${timeoutSeconds} seconds for ${numberOfMessages} message(s) from Kafka`), + timeoutSeconds * 1000 + ) + const onMessage = (payload: EachMessagePayload) => { + accumulator.push(parseRawEventMessage(payload.message)) + if (accumulator.length >= numberOfMessages) { + this.removeListener('message', onMessage) + resolve(accumulator) + } + } + this.addListener('message', onMessage) + }) + } +} From 9d526842d8311bd476dfbf90cab70fa76cdf705d Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 02:01:19 +0100 Subject: [PATCH 151/218] Clean up new testing system --- tests/clickhouse/process-event.ts | 19 ++----------------- tests/helpers/kafka.ts | 12 +++++++++--- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/tests/clickhouse/process-event.ts b/tests/clickhouse/process-event.ts index ee606ae5..2a0bd879 100644 --- a/tests/clickhouse/process-event.ts +++ b/tests/clickhouse/process-event.ts @@ -9,7 +9,7 @@ import { KafkaObserver } from '../helpers/kafka' let server: PluginsServer let closeServer: () => Promise -kafkaObserver = new KafkaObserver() +const kafkaObserver = new KafkaObserver() beforeEach(async () => { ;[server, closeServer] = await createServer() @@ -22,20 +22,5 @@ afterEach(() => { }) test('event is passed through', async () => { - const rows1 = await getPluginAttachmentRows(server) - expect(rows1).toEqual([ - { - content_type: 'application/octet-stream', - contents: Buffer.from([116, 101, 115, 116]), - file_name: 'test.txt', - file_size: 4, - id: 1, - key: 'maxmindMmdb', - plugin_config_id: 39, - team_id: 2, - }, - ]) - server.db.query("update posthog_team set plugins_opt_in='f'") - const rows2 = await getPluginAttachmentRows(server) - expect(rows2).toEqual([]) + expect(1).toEqual(1) }) diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 515748c7..6722987d 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -31,6 +31,7 @@ export class KafkaObserver extends EventEmitter { if (this.isStarted) {return} this.isStarted = true return await new Promise(async (resolve, reject) => { + await this.producer.connect() await this.consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) await this.consumer.run({ eachMessage: async (payload) => { @@ -50,10 +51,15 @@ export class KafkaObserver extends EventEmitter { await this.producer.disconnect() } - public handOffMessa + public async handOffMessage(message: EventMessage): Promise { + this.producer.send({ + topic: KAFKA_EVENTS_INGESTION_HANDOFF, + messages: [{ value: Buffer.from(JSON.stringify(message)) }], + }) + } public async waitForProcessedMessages(numberOfMessages: number): Promise { - return await new Promise((resolve, reject) => { + return await new Promise((resolve, reject) => { const accumulator: EventMessage[] = [] const timeoutSeconds = numberOfMessages * 2 // give every message 2 s on average to show up setTimeout( @@ -62,7 +68,7 @@ export class KafkaObserver extends EventEmitter { timeoutSeconds * 1000 ) const onMessage = (payload: EachMessagePayload) => { - accumulator.push(parseRawEventMessage(payload.message)) + accumulator.push(parseRawEventMessage(JSON.parse(payload.message.value!.toString()))) if (accumulator.length >= numberOfMessages) { this.removeListener('message', onMessage) resolve(accumulator) From 242d3d031ed9be0f555fc7a859bc44e26eaf12aa Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 02:04:38 +0100 Subject: [PATCH 152/218] Make prettier compatible with ESLint --- .eslintrc.js | 2 +- tests/helpers/kafka.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index c5c9d271..4ee0805d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,7 +1,7 @@ module.exports = { parser: '@typescript-eslint/parser', plugins: ['@typescript-eslint'], - extends: ['plugin:@typescript-eslint/recommended', 'prettier/@typescript-eslint'], + extends: ['plugin:@typescript-eslint/recommended', 'prettier', 'prettier/@typescript-eslint', 'prettier/react'], ignorePatterns: ['bin', 'dist', 'node_modules'], rules: { '@typescript-eslint/no-unused-vars': 'off', diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 6722987d..3a4e9a19 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -28,7 +28,9 @@ export class KafkaObserver extends EventEmitter { } public async start(): Promise { - if (this.isStarted) {return} + if (this.isStarted) { + return + } this.isStarted = true return await new Promise(async (resolve, reject) => { await this.producer.connect() From 21dc1a9222c31eb7fec83d13d9bcdc7ccbe944a1 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 10:26:22 +0100 Subject: [PATCH 153/218] Add compatibility with Plugin.latest_tag in test --- tests/sql.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/sql.test.ts b/tests/sql.test.ts index 050182c1..8b81be06 100644 --- a/tests/sql.test.ts +++ b/tests/sql.test.ts @@ -85,6 +85,8 @@ test('getPluginRows', async () => { from_web: false, id: 60, organization_id: organizationId, + latest_tag: null, + latest_tag_checked_at: null, name: 'test-maxmind-plugin', plugin_type: 'custom', source: null, From c759e42d149642ade2e18600db90682016bbd314 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 11:20:06 +0100 Subject: [PATCH 154/218] Fix minor issues --- src/ingestion/utils.ts | 1 + tests/clickhouse/process-event.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 1f78235e..8098cd1a 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -56,6 +56,7 @@ export function elementsToString(elements: Element[]): string { return ret.join(';') } +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types export function sanitizeEventName(eventName: any): string { if (typeof eventName !== 'string') { try { diff --git a/tests/clickhouse/process-event.ts b/tests/clickhouse/process-event.ts index 2a0bd879..e4e2a624 100644 --- a/tests/clickhouse/process-event.ts +++ b/tests/clickhouse/process-event.ts @@ -2,7 +2,7 @@ import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } import { PluginConfig, PluginError, PluginsServer } from '../../src/types' import { createServer } from '../../src/server' import { resetTestDatabase } from '../helpers/sql' -import { organizationId } from '../helpers/plugins' +import { commonOrganizationId } from '../helpers/plugins' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { Consumer, EachMessagePayload, Kafka, Producer } from 'kafkajs' import { KafkaObserver } from '../helpers/kafka' From 85b446270e8547aa9261cef200a92a9ad28a6522 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 11:29:15 +0100 Subject: [PATCH 155/218] Try not terminating program on Kafka consumer crash --- src/ingestion/kafka-queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 6eac4154..79178019 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -142,7 +142,7 @@ export class KafkaQueue implements Queue { consumer.on(CRASH, ({ payload: { error, groupId } }) => { status.error('โš ๏ธ', `Kafka consumer group ${groupId} crashed:\n`, error) Sentry.captureException(error) - killGracefully() + // killGracefully() }) consumer.on(CONNECT, () => { status.info('โœ…', 'Kafka consumer connected!') From c2901db33aea3210baeac4adf98cb238eee6c857 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 11:42:20 +0100 Subject: [PATCH 156/218] Change KAFKA_ADVERTISED_HOST_NAME to localhost --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96b84707..efb276c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,7 @@ jobs: ports: - '9092:9092' env: - KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_ADVERTISED_HOST_NAME: localhost KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 env: From 9d3c72e260ae83e009ad23655c81e1c4c61e40a0 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 12:05:21 +0100 Subject: [PATCH 157/218] Address feedback --- src/config.ts | 2 +- src/ingestion/kafka-queue.ts | 2 +- src/server.ts | 2 +- tests/postgres/plugins.test.ts | 12 ++++++------ tests/postgres/queue.test.ts | 12 ++++++------ tests/postgres/vm.test.ts | 14 +++++++------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/config.ts b/src/config.ts index 24e53d0a..6fbbec7e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -89,5 +89,5 @@ export function overrideWithEnv( } } } - return newConfig as PluginsServerConfig + return newConfig } diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 79178019..6eac4154 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -142,7 +142,7 @@ export class KafkaQueue implements Queue { consumer.on(CRASH, ({ payload: { error, groupId } }) => { status.error('โš ๏ธ', `Kafka consumer group ${groupId} crashed:\n`, error) Sentry.captureException(error) - // killGracefully() + killGracefully() }) consumer.on(CONNECT, () => { status.info('โœ…', 'Kafka consumer connected!') diff --git a/src/server.ts b/src/server.ts index 6c4fcc24..10f3cb44 100644 --- a/src/server.ts +++ b/src/server.ts @@ -188,9 +188,9 @@ export async function startPluginsServer( await stopFastifyInstance(fastifyInstance!) } await queue?.stop() - await pubSub?.quit() await kafkaProducer?.disconnect() pubSub?.disconnect() + await pubSub?.quit() pingJob && schedule.cancelJob(pingJob) statsJob && schedule.cancelJob(statsJob) await stopSchedule?.() diff --git a/tests/postgres/plugins.test.ts b/tests/postgres/plugins.test.ts index c8cc15ea..d577afec 100644 --- a/tests/postgres/plugins.test.ts +++ b/tests/postgres/plugins.test.ts @@ -1,6 +1,6 @@ -import { runPlugins, setupPlugins } from '../src/plugins' -import { createServer } from '../src/server' -import { LogLevel, PluginsServer } from '../src/types' +import { runPlugins, setupPlugins } from '../../src/plugins' +import { createServer } from '../../src/server' +import { LogLevel, PluginsServer } from '../../src/types' import { PluginEvent } from '@posthog/plugin-scaffold/src/types' import { mockPluginTempFolder, @@ -8,8 +8,8 @@ import { plugin60, pluginAttachment1, pluginConfig39, -} from './helpers/plugins' -import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } from './helpers/sqlMock' +} from '../helpers/plugins' +import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } from '../helpers/sqlMock' jest.mock('../src/sql') let mockServer: PluginsServer @@ -92,7 +92,7 @@ test('plugin returns null', async () => { test('plugin meta has what it should have', async () => { getPluginRows.mockReturnValueOnce([ mockPluginWithArchive(` - function setupPlugin (meta) { meta.global.key = 'value' } + function setupPlugin (meta) { meta.global.key = 'value' } function processEvent (event, meta) { event.properties=meta; return event } `), ]) diff --git a/tests/postgres/queue.test.ts b/tests/postgres/queue.test.ts index 78b003d8..16492522 100644 --- a/tests/postgres/queue.test.ts +++ b/tests/postgres/queue.test.ts @@ -1,9 +1,9 @@ -import { redisFactory } from './helpers/redis' -import { startQueue } from '../src/worker/queue' -import { createServer } from '../src/server' -import { LogLevel, PluginsServer } from '../src/types' -import Client from '../src/celery/client' -import { runPlugins } from '../src/plugins' +import { redisFactory } from '../helpers/redis' +import { startQueue } from '../../src/worker/queue' +import { createServer } from '../../src/server' +import { LogLevel, PluginsServer } from '../../src/types' +import Client from '../../src/celery/client' +import { runPlugins } from '../../src/plugins' jest.mock('ioredis', () => redisFactory()) diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index a7a681a8..4eccc751 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -1,12 +1,12 @@ -import { createPluginConfigVM } from '../src/vm' -import { PluginConfig, PluginsServer, Plugin } from '../src/types' +import { createPluginConfigVM } from '../../src/vm' +import { PluginsServer } from '../../src/types' import { PluginEvent } from '@posthog/plugin-scaffold' -import { createServer } from '../src/server' +import { createServer } from '../../src/server' import * as fetch from 'node-fetch' -import { delay } from '../src/utils' -import Client from '../src/celery/client' -import { resetTestDatabase } from './helpers/sql' -import { pluginConfig39 } from './helpers/plugins' +import { delay } from '../../src/utils' +import Client from '../../src/celery/client' +import { resetTestDatabase } from '../helpers/sql' +import { pluginConfig39 } from '../helpers/plugins' jest.mock('../src/celery/client') From e344542c45f543530241c5d0fc1154c879ce1b69 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 12:43:57 +0100 Subject: [PATCH 158/218] Separate Postgres and DB classes --- src/db.ts | 25 ++++++++++++++----------- src/server.ts | 21 +++++++++------------ src/types.ts | 16 +++++++++------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/db.ts b/src/db.ts index 0fac596f..b474faa4 100644 --- a/src/db.ts +++ b/src/db.ts @@ -2,26 +2,29 @@ import { Properties } from '@posthog/plugin-scaffold' import { ClickHouse } from 'clickhouse' import { Producer } from 'kafkajs' import { DateTime } from 'luxon' -import { Pool, PoolConfig } from 'pg' +import { Pool } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { unparsePersonPartial } from './ingestion/utils' import { Person, PersonDistinctId, RawPerson } from './types' import { castTimestampOrNow } from './utils' -export class DB extends Pool { +/** The recommended way of accessing the database. */ +export class DB { + /** Postgres connection pool for primary database access. */ + postgres: Pool /** Kafka producer used for syncing Postgres and ClickHouse person data. */ kafkaProducer?: Producer /** ClickHouse used for syncing Postgres and ClickHouse person data. */ clickhouse?: ClickHouse - constructor(config: PoolConfig, kafkaProducer?: Producer, clickhouse?: ClickHouse) { - super(config) + constructor(postgres: Pool, kafkaProducer?: Producer, clickhouse?: ClickHouse) { + this.postgres = postgres this.kafkaProducer = kafkaProducer this.clickhouse = clickhouse } public async fetchPerson(teamId: number, distinctId: string): Promise { - const selectResult = await this.query( + const selectResult = await this.postgres.query( `SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, @@ -47,7 +50,7 @@ export class DB extends Pool { isIdentified: boolean, uuid: string ): Promise { - const insertResult = await this.query( + const insertResult = await this.postgres.query( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', [createdAt.toISO(), JSON.stringify(properties), teamId, isUserId, isIdentified, uuid] ) @@ -70,7 +73,7 @@ export class DB extends Pool { public async updatePerson(person: Person, update: Partial): Promise { const updatedPerson: Person = { ...person, ...update } - await this.query( + await this.postgres.query( `UPDATE posthog_person SET ${Object.keys(update).map( (field, index) => field + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, @@ -93,8 +96,8 @@ export class DB extends Pool { } public async deletePerson(personId: number): Promise { - await this.query('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) - await this.query('DELETE FROM posthog_person WHERE id = $1', [personId]) + await this.postgres.query('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) + await this.postgres.query('DELETE FROM posthog_person WHERE id = $1', [personId]) if (this.clickhouse) { await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() await this.clickhouse @@ -104,7 +107,7 @@ export class DB extends Pool { } public async addDistinctId(person: Person, distinctId: string): Promise { - const insertResult = await this.query( + const insertResult = await this.postgres.query( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', [distinctId, person.id, person.team_id] ) @@ -122,7 +125,7 @@ export class DB extends Pool { update: Partial ): Promise { const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } - await this.query( + await this.postgres.query( `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( (field, index) => field + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, diff --git a/src/server.ts b/src/server.ts index 10f3cb44..6accd6ee 100644 --- a/src/server.ts +++ b/src/server.ts @@ -88,18 +88,15 @@ export async function createServer( kafkaProducer = kafka.producer() } - const db = new DB( - { - connectionString: serverConfig.DATABASE_URL, - ssl: process.env.DEPLOYMENT?.startsWith('Heroku') - ? { - rejectUnauthorized: false, - } - : undefined, - }, - kafkaProducer, - clickhouse - ) + const postgres = new Pool({ + connectionString: serverConfig.DATABASE_URL, + ssl: process.env.DEPLOYMENT?.startsWith('Heroku') + ? { + rejectUnauthorized: false, + } + : undefined, + }) + const db = new DB(postgres, kafkaProducer, clickhouse) let statsd: StatsD | undefined if (serverConfig.STATSD_HOST) { diff --git a/src/types.ts b/src/types.ts index 5a6576dc..3a734f6b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -196,13 +196,6 @@ export interface EventMessage extends BaseEventMessage { sent_at: DateTime | null } -export interface EventData extends PluginEvent { - properties?: Properties - timestamp?: string - $set?: Properties - offset?: number -} - /** Usable Team model. */ export interface Team { id: number @@ -272,3 +265,12 @@ export interface CohortPeople { cohort_id: number person_id: number } +export interface SessionRecordingEvent { + uuid: string + timestamp: string + team_id: number + distinct_id: string + session_id: string + snapshot_data: string + created_at: string +} From 0b0ba88f0df887b508a6fbb43ee928a09a254fc2 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 12:49:10 +0100 Subject: [PATCH 159/218] Consume from kafka_events topic in KafkaObserver --- tests/helpers/kafka.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 3a4e9a19..66d0fab0 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -1,8 +1,8 @@ import { EventEmitter } from 'events' import { Kafka, Consumer, logLevel, EachMessagePayload, Producer } from 'kafkajs' -import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' +import { KAFKA_EVENTS, KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' import { parseRawEventMessage } from '../../src/ingestion/utils' -import { EventMessage, RawEventMessage } from '../../src/types' +import { EventMessage } from '../../src/types' import { UUIDT } from '../../src/utils' export class KafkaObserver extends EventEmitter { @@ -34,7 +34,7 @@ export class KafkaObserver extends EventEmitter { this.isStarted = true return await new Promise(async (resolve, reject) => { await this.producer.connect() - await this.consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) + await this.consumer.subscribe({ topic: KAFKA_EVENTS }) await this.consumer.run({ eachMessage: async (payload) => { this.emit('message', payload) From f58344807beb8b1c3cca44bbbb4d02c109dec77f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 12:49:23 +0100 Subject: [PATCH 160/218] Fix process-event test file name --- tests/clickhouse/{process-event.ts => process-event.test.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/clickhouse/{process-event.ts => process-event.test.ts} (100%) diff --git a/tests/clickhouse/process-event.ts b/tests/clickhouse/process-event.test.ts similarity index 100% rename from tests/clickhouse/process-event.ts rename to tests/clickhouse/process-event.test.ts From b02c08aa45eeb5cc535ed7cb5a65c64091492808 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 12:49:53 +0100 Subject: [PATCH 161/218] Check for UUID in handed off events --- src/worker/queue.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/worker/queue.ts b/src/worker/queue.ts index a323e471..99ae5ed2 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -73,7 +73,11 @@ async function startQueueKafka( ): Promise { const kafkaQueue = new KafkaQueue(server, processEventBatch, async (event: PluginEvent) => { const { distinct_id, ip, site_url, team_id, now, sent_at, uuid } = event - await server.eventsProcessor.processEventEE( + if (!uuid) { + status.error('โ“', 'UUID missing in event received from Kafka!') + return + } + await server.eventsProcessor.processEvent( distinct_id, ip, site_url, @@ -81,7 +85,7 @@ async function startQueueKafka( team_id, DateTime.fromISO(now), sent_at ? DateTime.fromISO(sent_at) : null, - uuid! + uuid ) }) From e9000d02c238d91413d9ba2b9227b494ae822a72 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 12:50:26 +0100 Subject: [PATCH 162/218] Return from EventsProcessor.processEvent --- src/ingestion/process-event.ts | 38 ++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 954448f0..6b77ade3 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,8 +1,8 @@ import { PluginEvent, Properties } from '@posthog/plugin-scaffold' import { DateTime, Duration } from 'luxon' -import { PluginsServer, EventData, Element, Team, Person, PersonDistinctId, CohortPeople } from '../types' +import { PluginsServer, Element, Team, Person, PersonDistinctId, CohortPeople, SessionRecordingEvent } from '../types' import { castTimestampOrNow, UUIDT } from '../utils' -import { Event as EventProto } from '../idl/protos' +import { Event as EventProto, IEvent } from '../idl/protos' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { sanitizeEventName, elementsToString, unparsePersonPartial } from './utils' @@ -22,7 +22,7 @@ export class EventsProcessor { this.kafkaProducer = pluginsServer.kafkaProducer! } - public async processEventEE( + public async processEvent( distinctId: string, ip: string, siteUrl: string, @@ -31,7 +31,7 @@ export class EventsProcessor { now: DateTime, sentAt: DateTime | null, eventUuid: string - ): Promise { + ): Promise { const singleSaveTimer = new Date() const properties: Properties = data.properties ?? {} @@ -47,8 +47,10 @@ export class EventsProcessor { const ts = this.handleTimestamp(data, now, sentAt) this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) + let result: IEvent | SessionRecordingEvent + if (data['event'] === '$snapshot') { - await this.createSessionRecordingEvent( + result = await this.createSessionRecordingEvent( eventUuid, teamId, distinctId, @@ -58,7 +60,7 @@ export class EventsProcessor { ) this.pluginsServer.statsd?.timing('kafka_queue.single_save.snapshot', singleSaveTimer) } else { - await this.captureEE( + result = await this.captureEE( eventUuid, personUuid, ip, @@ -72,9 +74,11 @@ export class EventsProcessor { ) this.pluginsServer.statsd?.timing('kafka_queue.single_save.standard', singleSaveTimer) } + + return result } - private handleTimestamp(data: EventData, now: DateTime, sentAt: DateTime | null): DateTime { + private handleTimestamp(data: PluginEvent, now: DateTime, sentAt: DateTime | null): DateTime { if (data['timestamp']) { if (sentAt) { // sent_at - timestamp == now - x @@ -288,7 +292,7 @@ export class EventsProcessor { properties: Properties, timestamp: DateTime, sentAt: DateTime | null - ): Promise { + ): Promise { event = sanitizeEventName(event) const elements: Record[] | undefined = properties['$elements'] @@ -336,7 +340,7 @@ export class EventsProcessor { } catch {} } - await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elements_list) + return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elements_list) } private async storeNamesAndProperties(team: Team, event: string, properties: Properties): Promise { @@ -395,11 +399,11 @@ export class EventsProcessor { properties?: Properties, timestamp?: DateTime | string, elements?: Element[] - ): Promise { + ): Promise { const timestampString = castTimestampOrNow(timestamp) const elementsChain = elements && elements.length ? elementsToString(elements) : '' - const message = EventProto.create({ + const data: IEvent = { uuid, event, properties: JSON.stringify(properties ?? {}), @@ -408,12 +412,14 @@ export class EventsProcessor { distinctId, elementsChain, createdAt: timestampString, - }) + } await this.kafkaProducer.send({ topic: KAFKA_EVENTS, - messages: [{ key: uuid, value: EventProto.encodeDelimited(message).finish() as Buffer }], + messages: [{ key: uuid, value: EventProto.encodeDelimited(EventProto.create(data)).finish() as Buffer }], }) + + return data } private async createSessionRecordingEvent( @@ -423,10 +429,10 @@ export class EventsProcessor { session_id: string, timestamp: DateTime | string, snapshot_data: Record - ): Promise { + ): Promise { const timestampString = castTimestampOrNow(timestamp) - const data = { + const data: SessionRecordingEvent = { uuid, team_id: team_id, distinct_id: distinct_id, @@ -440,5 +446,7 @@ export class EventsProcessor { topic: KAFKA_SESSION_RECORDING_EVENTS, messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }], }) + + return data } } From 2199a795d0f64bf4e8b7bf562ae6055ace2e36a4 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 26 Jan 2021 15:16:39 +0100 Subject: [PATCH 163/218] use `db.postgresQuery` instead of vague `db.query` --- src/db.ts | 23 +++++++++++++++-------- src/extensions/posthog.ts | 2 +- src/extensions/storage.ts | 16 ++++++++-------- src/ingestion/process-event.ts | 20 +++++++++++--------- src/server.ts | 1 + src/sql.ts | 8 ++++---- src/types.ts | 1 + tests/postgres/vm.test.ts | 2 +- tests/sql.test.ts | 12 ++++++------ 9 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/db.ts b/src/db.ts index b474faa4..9f16ff59 100644 --- a/src/db.ts +++ b/src/db.ts @@ -2,7 +2,7 @@ import { Properties } from '@posthog/plugin-scaffold' import { ClickHouse } from 'clickhouse' import { Producer } from 'kafkajs' import { DateTime } from 'luxon' -import { Pool } from 'pg' +import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { unparsePersonPartial } from './ingestion/utils' import { Person, PersonDistinctId, RawPerson } from './types' @@ -23,8 +23,15 @@ export class DB { this.clickhouse = clickhouse } + public async postgresQuery( + queryTextOrConfig: string | QueryConfig, + values?: I + ): Promise> { + return this.postgres.query(queryTextOrConfig, values) + } + public async fetchPerson(teamId: number, distinctId: string): Promise { - const selectResult = await this.postgres.query( + const selectResult = await this.postgresQuery( `SELECT posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, @@ -50,7 +57,7 @@ export class DB { isIdentified: boolean, uuid: string ): Promise { - const insertResult = await this.postgres.query( + const insertResult = await this.postgresQuery( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', [createdAt.toISO(), JSON.stringify(properties), teamId, isUserId, isIdentified, uuid] ) @@ -73,7 +80,7 @@ export class DB { public async updatePerson(person: Person, update: Partial): Promise { const updatedPerson: Person = { ...person, ...update } - await this.postgres.query( + await this.postgresQuery( `UPDATE posthog_person SET ${Object.keys(update).map( (field, index) => field + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, @@ -96,8 +103,8 @@ export class DB { } public async deletePerson(personId: number): Promise { - await this.postgres.query('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) - await this.postgres.query('DELETE FROM posthog_person WHERE id = $1', [personId]) + await this.postgresQuery('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) + await this.postgresQuery('DELETE FROM posthog_person WHERE id = $1', [personId]) if (this.clickhouse) { await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() await this.clickhouse @@ -107,7 +114,7 @@ export class DB { } public async addDistinctId(person: Person, distinctId: string): Promise { - const insertResult = await this.postgres.query( + const insertResult = await this.postgresQuery( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', [distinctId, person.id, person.team_id] ) @@ -125,7 +132,7 @@ export class DB { update: Partial ): Promise { const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } - await this.postgres.query( + await this.postgresQuery( `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( (field, index) => field + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index dd1ab85e..51a09b04 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -1,7 +1,7 @@ import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../ingestion/topics' import { Properties } from '@posthog/plugin-scaffold' import { DateTime } from 'luxon' -import { PluginsServer, PluginConfig, RawEventMessage, EventData } from 'types' +import { PluginsServer, PluginConfig, RawEventMessage } from 'types' import { version } from '../../package.json' import Client from '../celery/client' import { UUIDT } from '../utils' diff --git a/src/extensions/storage.ts b/src/extensions/storage.ts index 8fad920a..9246e61e 100644 --- a/src/extensions/storage.ts +++ b/src/extensions/storage.ts @@ -3,7 +3,7 @@ import { StorageExtension } from '@posthog/plugin-scaffold' export function createStorage(server: PluginsServer, pluginConfig: PluginConfig): StorageExtension { const get = async function (key: string, defaultValue: unknown): Promise { - const result = await server.db.query( + const result = await server.db.postgresQuery( 'SELECT * FROM posthog_pluginstorage WHERE "plugin_config_id"=$1 AND "key"=$2 LIMIT 1', [pluginConfig.id, key] ) @@ -11,16 +11,16 @@ export function createStorage(server: PluginsServer, pluginConfig: PluginConfig) } const set = async function (key: string, value: unknown): Promise { if (typeof value === 'undefined') { - await server.db.query('DELETE FROM posthog_pluginstorage WHERE "plugin_config_id"=$1 AND "key"=$2', [ - pluginConfig.id, - key, - ]) + await server.db.postgresQuery( + 'DELETE FROM posthog_pluginstorage WHERE "plugin_config_id"=$1 AND "key"=$2', + [pluginConfig.id, key] + ) } else { - await server.db.query( + await server.db.postgresQuery( ` - INSERT INTO posthog_pluginstorage ("plugin_config_id", "key", "value") + INSERT INTO posthog_pluginstorage ("plugin_config_id", "key", "value") VALUES ($1, $2, $3) - ON CONFLICT ("plugin_config_id", "key") + ON CONFLICT ("plugin_config_id", "key") DO UPDATE SET value = $3 `, [pluginConfig.id, key, JSON.stringify(value)] diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 6b77ade3..09518c5c 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -258,20 +258,22 @@ export class EventsProcessor { // merge the distinct_ids for (const other_person of peopleToMerge) { const other_person_distinct_ids: PersonDistinctId[] = ( - await this.db.query('SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', [ - other_person, - mergeInto.team_id, - ]) + await this.db.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', + [other_person, mergeInto.team_id] + ) ).rows for (const person_distinct_id of other_person_distinct_ids) { await this.db.updateDistinctId(person_distinct_id, { person_id: mergeInto.id }) } const other_person_cohort_ids: CohortPeople[] = ( - await this.db.query('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [other_person.id]) + await this.db.postgresQuery('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [ + other_person.id, + ]) ).rows for (const person_cohort_id of other_person_cohort_ids) { - await this.db.query('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ + await this.db.postgresQuery('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ mergeInto.id, person_cohort_id.id, ]) @@ -311,7 +313,7 @@ export class EventsProcessor { })) } - const team: Team = (await this.db.query('SELECT * FROM posthog_team WHERE id = $1', [teamId])).rows[0] + const team: Team = (await this.db.postgresQuery('SELECT * FROM posthog_team WHERE id = $1', [teamId])).rows[0] if (!team.anonymize_ips && !('$ip' in properties)) { properties['$ip'] = ip @@ -319,7 +321,7 @@ export class EventsProcessor { this.storeNamesAndProperties(team, event, properties) - const pdiSelectResult = await this.db.query( + const pdiSelectResult = await this.db.postgresQuery( 'SELECT COUNT(*) AS pdicount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', [teamId, distinctId] ) @@ -373,7 +375,7 @@ export class EventsProcessor { } } if (save) { - await this.db.query( + await this.db.postgresQuery( `UPDATE posthog_team SET ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, event_properties_with_usage = $5, event_properties_numerical = $6 diff --git a/src/server.ts b/src/server.ts index 6accd6ee..a9963c9b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -117,6 +117,7 @@ export async function createServer( const server: Omit = { ...serverConfig, db, + postgres, redis, clickhouse, kafka, diff --git a/src/sql.ts b/src/sql.ts index 78b262b2..c247a3e0 100644 --- a/src/sql.ts +++ b/src/sql.ts @@ -1,7 +1,7 @@ import { Plugin, PluginAttachmentDB, PluginConfig, PluginConfigId, PluginError, PluginsServer } from './types' export async function getPluginRows(server: PluginsServer): Promise { - const { rows: pluginRows }: { rows: Plugin[] } = await server.db.query( + const { rows: pluginRows }: { rows: Plugin[] } = await server.db.postgresQuery( `SELECT posthog_plugin.* FROM posthog_plugin WHERE id in (SELECT posthog_pluginconfig.plugin_id FROM posthog_pluginconfig @@ -13,7 +13,7 @@ export async function getPluginRows(server: PluginsServer): Promise { } export async function getPluginAttachmentRows(server: PluginsServer): Promise { - const { rows }: { rows: PluginAttachmentDB[] } = await server.db.query( + const { rows }: { rows: PluginAttachmentDB[] } = await server.db.postgresQuery( `SELECT posthog_pluginattachment.* FROM posthog_pluginattachment WHERE plugin_config_id in (SELECT posthog_pluginconfig.id FROM posthog_pluginconfig @@ -24,7 +24,7 @@ export async function getPluginAttachmentRows(server: PluginsServer): Promise { - const { rows }: { rows: PluginConfig[] } = await server.db.query( + const { rows }: { rows: PluginConfig[] } = await server.db.postgresQuery( `SELECT posthog_pluginconfig.* FROM posthog_pluginconfig LEFT JOIN posthog_team ON posthog_team.id = posthog_pluginconfig.team_id @@ -38,7 +38,7 @@ export async function setError( pluginError: PluginError | null, pluginConfig: PluginConfig | PluginConfigId ): Promise { - await server.db.query('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ + await server.db.postgresQuery('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ pluginError, typeof pluginConfig === 'object' ? pluginConfig?.id : pluginConfig, ]) diff --git a/src/types.ts b/src/types.ts index 3a734f6b..6a49fc31 100644 --- a/src/types.ts +++ b/src/types.ts @@ -52,6 +52,7 @@ export interface PluginsServerConfig extends Record { export interface PluginsServer extends PluginsServerConfig { // active connections to Postgres, Redis, ClickHouse, Kafka, StatsD db: DB + postgres: Pool redis: Redis clickhouse?: ClickHouse kafka?: Kafka diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index 4eccc751..8d48cdec 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -28,7 +28,7 @@ beforeEach(async () => { afterEach(async () => { mockServer.redis.disconnect() - await mockServer.db.end() + await mockServer.postgres.end() jest.clearAllMocks() }) diff --git a/tests/sql.test.ts b/tests/sql.test.ts index 6315c4ce..5708013d 100644 --- a/tests/sql.test.ts +++ b/tests/sql.test.ts @@ -28,7 +28,7 @@ test('getPluginAttachmentRows', async () => { team_id: 2, }, ]) - server.db.query("update posthog_team set plugins_opt_in='f'") + server.db.postgresQuery("update posthog_team set plugins_opt_in='f'") const rows2 = await getPluginAttachmentRows(server) expect(rows2).toEqual([]) }) @@ -49,7 +49,7 @@ test('getPluginConfigRows', async () => { team_id: 2, }, ]) - server.db.query("update posthog_team set plugins_opt_in='f'") + server.db.postgresQuery("update posthog_team set plugins_opt_in='f'") const rows2 = await getPluginConfigRows(server) expect(rows2).toEqual([]) }) @@ -94,7 +94,7 @@ test('getPluginRows', async () => { url: 'https://www.npmjs.com/package/posthog-maxmind-plugin', }, ]) - server.db.query("update posthog_team set plugins_opt_in='f'") + server.db.postgresQuery("update posthog_team set plugins_opt_in='f'") const rows2 = await getPluginRows(server) expect(rows2).toEqual([]) }) @@ -109,17 +109,17 @@ test('setError', async () => { config: {}, error: undefined, } - server.db.query = jest.fn() as any + server.db.postgresQuery = jest.fn() as any await setError(server, null, pluginConfig39) - expect(server.db.query).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ + expect(server.db.postgresQuery).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ null, pluginConfig39.id, ]) const pluginError: PluginError = { message: 'error happened', time: 'now' } await setError(server, pluginError, pluginConfig39) - expect(server.db.query).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ + expect(server.db.postgresQuery).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ pluginError, pluginConfig39.id, ]) From 510b2d7366c40260abf6f1815a5088228b420069 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 15:22:53 +0100 Subject: [PATCH 164/218] Run the plugins suite universally --- tests/{postgres => }/plugins.test.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{postgres => }/plugins.test.ts (100%) diff --git a/tests/postgres/plugins.test.ts b/tests/plugins.test.ts similarity index 100% rename from tests/postgres/plugins.test.ts rename to tests/plugins.test.ts From 18e58ef79ceab80381cf4f2269ad46c24e05d0f8 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 15:26:13 +0100 Subject: [PATCH 165/218] Fix closeServer --- src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index a9963c9b..7d2a2510 100644 --- a/src/server.ts +++ b/src/server.ts @@ -136,7 +136,7 @@ export async function createServer( const closeServer = async () => { await server.redis.quit() - await server.db.end() + await server.postgres.end() } return [server as PluginsServer, closeServer] From ccf31bcaf6437299aa2b5e659aa27edce4c71f10 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 26 Jan 2021 15:37:41 +0100 Subject: [PATCH 166/218] javaScriptVariableCase --- src/ingestion/process-event.ts | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 09518c5c..0ba729dc 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -242,44 +242,42 @@ export class EventsProcessor { } private async mergePeople(mergeInto: Person, peopleToMerge: Person[]): Promise { - let first_seen = mergeInto.created_at + let firstSeen = mergeInto.created_at // merge the properties - for (const other_person of peopleToMerge) { - mergeInto.properties = { ...other_person.properties, ...mergeInto.properties } - if (other_person.created_at < first_seen) { + for (const otherPerson of peopleToMerge) { + mergeInto.properties = { ...otherPerson.properties, ...mergeInto.properties } + if (otherPerson.created_at < firstSeen) { // Keep the oldest created_at (i.e. the first time we've seen this person) - first_seen = other_person.created_at + firstSeen = otherPerson.created_at } } - await this.db.updatePerson(mergeInto, { created_at: first_seen }) + await this.db.updatePerson(mergeInto, { created_at: firstSeen }) // merge the distinct_ids - for (const other_person of peopleToMerge) { - const other_person_distinct_ids: PersonDistinctId[] = ( + for (const otherPerson of peopleToMerge) { + const otherPersonDistinctIds: PersonDistinctId[] = ( await this.db.postgresQuery( 'SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', - [other_person, mergeInto.team_id] + [otherPerson, mergeInto.team_id] ) ).rows - for (const person_distinct_id of other_person_distinct_ids) { - await this.db.updateDistinctId(person_distinct_id, { person_id: mergeInto.id }) + for (const personDistinctId of otherPersonDistinctIds) { + await this.db.updateDistinctId(personDistinctId, { person_id: mergeInto.id }) } - const other_person_cohort_ids: CohortPeople[] = ( - await this.db.postgresQuery('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [ - other_person.id, - ]) + const otherPersonCohortIds: CohortPeople[] = ( + await this.db.postgresQuery('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [otherPerson.id]) ).rows - for (const person_cohort_id of other_person_cohort_ids) { + for (const personCohortId of otherPersonCohortIds) { await this.db.postgresQuery('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ mergeInto.id, - person_cohort_id.id, + personCohortId.id, ]) } - await this.db.deletePerson(other_person.id) + await this.db.deletePerson(otherPerson.id) } } From 2dd58ef272ba6079185a397f3cb8dd237953b2b5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 15:38:58 +0100 Subject: [PATCH 167/218] Capture handleTimestamp error with Sentry --- src/ingestion/process-event.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 0ba729dc..98c3d899 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -4,10 +4,12 @@ import { PluginsServer, Element, Team, Person, PersonDistinctId, CohortPeople, S import { castTimestampOrNow, UUIDT } from '../utils' import { Event as EventProto, IEvent } from '../idl/protos' import { Producer } from 'kafkajs' -import { KAFKA_EVENTS, KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID, KAFKA_SESSION_RECORDING_EVENTS } from './topics' -import { sanitizeEventName, elementsToString, unparsePersonPartial } from './utils' +import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' +import { sanitizeEventName, elementsToString } from './utils' import { ClickHouse } from 'clickhouse' import { DB } from '../db' +import { status } from '../status' +import * as Sentry from '@sentry/node' export class EventsProcessor { pluginsServer: PluginsServer @@ -88,7 +90,8 @@ export class EventsProcessor { // otherwise we can't get a diff to add to now return now.plus(DateTime.fromISO(data['timestamp']).diff(sentAt)) } catch (error) { - console.error(error) + status.error('โš ๏ธ', 'Error when handling timestamp:', error) + Sentry.captureException(error) } } return DateTime.fromISO(data['timestamp']) From c6cedd26ca9588a8d1223a2440c1298d077dd426 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 15:46:26 +0100 Subject: [PATCH 168/218] Hand off message in test --- tests/clickhouse/process-event.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index e4e2a624..6734cb2a 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -6,6 +6,8 @@ import { commonOrganizationId } from '../helpers/plugins' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { Consumer, EachMessagePayload, Kafka, Producer } from 'kafkajs' import { KafkaObserver } from '../helpers/kafka' +import { UUIDT } from '../../src/utils' +import { DateTime } from 'luxon' let server: PluginsServer let closeServer: () => Promise @@ -22,5 +24,25 @@ afterEach(() => { }) test('event is passed through', async () => { + const uuid = new UUIDT().toString() + const now = new DateTime() + kafkaObserver.handOffMessage({ + distinct_id: 'abcd', + ip: '1.1.1.1', + site_url: 'x.com', + team_id: 1, + uuid, + data: { + distinct_id: 'abcd', + ip: '1.1.1.1', + site_url: 'x.com', + team_id: 1, + now: now.toString(), + event: 'test', + uuid, + }, + now, + sent_at: null, + }) expect(1).toEqual(1) }) From ec8ebb5c0b769f7a8d1b8247245d449c92b27a7f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 15:48:03 +0100 Subject: [PATCH 169/218] Fix plugins suite imports --- tests/plugins.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/plugins.test.ts b/tests/plugins.test.ts index d577afec..1c98ac6e 100644 --- a/tests/plugins.test.ts +++ b/tests/plugins.test.ts @@ -1,6 +1,6 @@ -import { runPlugins, setupPlugins } from '../../src/plugins' -import { createServer } from '../../src/server' -import { LogLevel, PluginsServer } from '../../src/types' +import { runPlugins, setupPlugins } from '../src/plugins' +import { createServer } from '../src/server' +import { LogLevel, PluginsServer } from '../src/types' import { PluginEvent } from '@posthog/plugin-scaffold/src/types' import { mockPluginTempFolder, @@ -8,8 +8,8 @@ import { plugin60, pluginAttachment1, pluginConfig39, -} from '../helpers/plugins' -import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } from '../helpers/sqlMock' +} from './helpers/plugins' +import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } from './helpers/sqlMock' jest.mock('../src/sql') let mockServer: PluginsServer From 966056fe8b1479c85cdcd0494df5ce58e50f103a Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Tue, 26 Jan 2021 16:09:50 +0100 Subject: [PATCH 170/218] Run the vm suite universally --- tests/{postgres => }/vm.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) rename tests/{postgres => }/vm.test.ts (98%) diff --git a/tests/postgres/vm.test.ts b/tests/vm.test.ts similarity index 98% rename from tests/postgres/vm.test.ts rename to tests/vm.test.ts index 8d48cdec..48fc0dc7 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/vm.test.ts @@ -1,12 +1,12 @@ -import { createPluginConfigVM } from '../../src/vm' -import { PluginsServer } from '../../src/types' +import { createPluginConfigVM } from '../src/vm' +import { PluginsServer } from '../src/types' import { PluginEvent } from '@posthog/plugin-scaffold' -import { createServer } from '../../src/server' +import { createServer } from '../src/server' import * as fetch from 'node-fetch' -import { delay } from '../../src/utils' -import Client from '../../src/celery/client' -import { resetTestDatabase } from '../helpers/sql' -import { pluginConfig39 } from '../helpers/plugins' +import { delay } from '../src/utils' +import Client from '../src/celery/client' +import { resetTestDatabase } from './helpers/sql' +import { pluginConfig39 } from './helpers/plugins' jest.mock('../src/celery/client') From 0eed9c28d307a38e41a4c2e42dc02ed817d2321c Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 26 Jan 2021 16:33:32 +0100 Subject: [PATCH 171/218] more javaScriptVariableCase --- src/ingestion/process-event.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 0ba729dc..46a6f2ca 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -267,13 +267,13 @@ export class EventsProcessor { await this.db.updateDistinctId(personDistinctId, { person_id: mergeInto.id }) } - const otherPersonCohortIds: CohortPeople[] = ( + const otherCohortPeople: CohortPeople[] = ( await this.db.postgresQuery('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [otherPerson.id]) ).rows - for (const personCohortId of otherPersonCohortIds) { + for (const cohortPeople of otherCohortPeople) { await this.db.postgresQuery('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ mergeInto.id, - personCohortId.id, + cohortPeople.id, ]) } @@ -296,14 +296,14 @@ export class EventsProcessor { event = sanitizeEventName(event) const elements: Record[] | undefined = properties['$elements'] - let elements_list: Element[] = [] + let elementsList: Element[] = [] if (elements && elements.length) { delete properties['$elements'] - elements_list = elements.map((el) => ({ - text: el['$el_text'] ? el['$el_text'].slice(0, 400) : null, + elementsList = elements.map((el) => ({ + text: el['$el_text']?.slice(0, 400), tag_name: el['tag_name'], - href: el['attr__href'] ? el['attr__href'].slice(0, 2048) : null, - attr_class: el['attr__class'] ? el['attr__class'].split(' ') : null, + href: el['attr__href']?.slice(0, 2048), + attr_class: el['attr__class']?.split(' '), attr_id: el['attr__id'], nth_child: el['nth_child'], nth_of_type: el['nth_of_type'], @@ -340,7 +340,7 @@ export class EventsProcessor { } catch {} } - return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elements_list) + return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList) } private async storeNamesAndProperties(team: Team, event: string, properties: Properties): Promise { From e9f6134d74da51874e6c4f98c93f4c8d1c65016f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 26 Jan 2021 16:53:48 +0100 Subject: [PATCH 172/218] disconnect without waiting for pending replies (.diconnect() instead of .quit()) --- src/server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 7d2a2510..48dd0e79 100644 --- a/src/server.ts +++ b/src/server.ts @@ -188,7 +188,6 @@ export async function startPluginsServer( await queue?.stop() await kafkaProducer?.disconnect() pubSub?.disconnect() - await pubSub?.quit() pingJob && schedule.cancelJob(pingJob) statsJob && schedule.cancelJob(statsJob) await stopSchedule?.() From e0645807304caf20ccb68b5dc00ea4db49888233 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 26 Jan 2021 17:02:30 +0100 Subject: [PATCH 173/218] dot quit seems to work better? --- src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 48dd0e79..781b2341 100644 --- a/src/server.ts +++ b/src/server.ts @@ -187,7 +187,7 @@ export async function startPluginsServer( } await queue?.stop() await kafkaProducer?.disconnect() - pubSub?.disconnect() + await pubSub?.quit() pingJob && schedule.cancelJob(pingJob) statsJob && schedule.cancelJob(statsJob) await stopSchedule?.() From 42975521e9b98b8327ee0d7b510b2060ac90f6bf Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 01:58:01 +0100 Subject: [PATCH 174/218] Add sanitizeSqlIdentifier --- src/db.ts | 6 +++--- src/utils.ts | 5 +++++ tests/utils.test.ts | 11 +++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/db.ts b/src/db.ts index 9f16ff59..a0d1ee83 100644 --- a/src/db.ts +++ b/src/db.ts @@ -4,7 +4,7 @@ import { Producer } from 'kafkajs' import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' -import { unparsePersonPartial } from './ingestion/utils' +import { sanitizeSqlIdentifier, unparsePersonPartial } from './ingestion/utils' import { Person, PersonDistinctId, RawPerson } from './types' import { castTimestampOrNow } from './utils' @@ -82,7 +82,7 @@ export class DB { const updatedPerson: Person = { ...person, ...update } await this.postgresQuery( `UPDATE posthog_person SET ${Object.keys(update).map( - (field, index) => field + ' = $' + (index + 1) + (field, index) => sanitizeSqlIdentifier(field) + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, [...Object.values(unparsePersonPartial(update)), person.id] ) @@ -134,7 +134,7 @@ export class DB { const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } await this.postgresQuery( `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( - (field, index) => field + ' = $' + (index + 1) + (field, index) => sanitizeSqlIdentifier(field) + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, [...Object.values(update), personDistinctId.id] ) diff --git a/src/utils.ts b/src/utils.ts index 42776b90..369c5836 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -294,3 +294,8 @@ export function delay(ms: number): Promise { setTimeout(resolve, ms) }) } + +/** Remove all quotes from the provided identifier to prevent SQL injection. */ +export function sanitizeSqlIdentifier(unquotedIdentifier: string): string { + return '"' + unquotedIdentifier.replace(/["']/g, '') + '"' +} diff --git a/tests/utils.test.ts b/tests/utils.test.ts index f9fc5011..56c5f8e9 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -7,6 +7,7 @@ import { cloneObject, UUID, UUIDT, + sanitizeSqlIdentifier, } from '../src/utils' import { randomBytes } from 'crypto' import { LogLevel } from '../src/types' @@ -295,3 +296,13 @@ describe('UUIDT', () => { expect(uuidtString.slice(14, 18)).toEqual('0000') }) }) + +describe('sanitizeSqlIdentifier', () => { + it('removes quotes inside and adds quotes around identifier', () => { + const rawIdentifier = 'some_field"; DROP TABLE actually_an_injection;' + + const sanitizedIdentifier = sanitizeSqlIdentifier(rawIdentifier) + + expect(sanitizeSqlIdentifier).toStrictEqual('"some_field; DROP TABLE actually_an_injection;"') + }) +}) From 381dfff2d81ba69c3c5cfa8858bd20e7aa236636 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 01:58:15 +0100 Subject: [PATCH 175/218] Fix updating person properties --- src/ingestion/process-event.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 5f769cca..b6e8ae6b 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -154,7 +154,7 @@ export class EventsProcessor { distinctId: string, properties: Properties, setOnce = false - ): Promise { + ): Promise { let personFound = await this.db.fetchPerson(teamId, distinctId) if (!personFound) { try { @@ -173,10 +173,10 @@ export class EventsProcessor { personFound = await this.db.fetchPerson(teamId, distinctId) } } - this.db.updatePerson( - personFound!, - setOnce ? { ...properties, ...personFound!.properties } : { ...personFound!.properties, ...properties } - ) + const updatedProperties: Properties = setOnce + ? { ...properties, ...personFound!.properties } + : { ...personFound!.properties, ...properties } + return await this.db.updatePerson(personFound!, { properties: updatedProperties }) } private async alias( @@ -314,7 +314,8 @@ export class EventsProcessor { })) } - const team: Team = (await this.db.postgresQuery('SELECT * FROM posthog_team WHERE id = $1', [teamId])).rows[0] + const teamQueryResult = await this.db.postgresQuery('SELECT * FROM posthog_team WHERE id = $1', [teamId]) + const team: Team = teamQueryResult.rows[0] if (!team.anonymize_ips && !('$ip' in properties)) { properties['$ip'] = ip From b4e66c56587cee6ed4b48a6778aca01eb7f3f05c Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 02:11:16 +0100 Subject: [PATCH 176/218] Fix problems --- src/db.ts | 4 ++-- src/ingestion/kafka-queue.ts | 1 - src/worker/piscina.js | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/db.ts b/src/db.ts index a0d1ee83..3659c7be 100644 --- a/src/db.ts +++ b/src/db.ts @@ -4,9 +4,9 @@ import { Producer } from 'kafkajs' import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' -import { sanitizeSqlIdentifier, unparsePersonPartial } from './ingestion/utils' +import { unparsePersonPartial } from './ingestion/utils' import { Person, PersonDistinctId, RawPerson } from './types' -import { castTimestampOrNow } from './utils' +import { castTimestampOrNow, sanitizeSqlIdentifier } from './utils' /** The recommended way of accessing the database. */ export class DB { diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 6eac4154..e5d68be3 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -4,7 +4,6 @@ import { PluginsServer, Queue, RawEventMessage } from 'types' import { KAFKA_EVENTS_INGESTION_HANDOFF } from './topics' import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' -import { parseRawEventMessage } from './utils' import { killGracefully } from '../utils' export type BatchCallback = (messages: Message[]) => Promise diff --git a/src/worker/piscina.js b/src/worker/piscina.js index 03617890..7287bfe3 100644 --- a/src/worker/piscina.js +++ b/src/worker/piscina.js @@ -1,5 +1,6 @@ const Sentry = require('@sentry/node') const { isMainThread, threadId } = require('worker_threads') +const { status } = require('../status') if (isMainThread) { const Piscina = require('piscina') From 53bfc07e6344470107e2130aabc83d0ce080d4d5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 02:21:51 +0100 Subject: [PATCH 177/218] Don't use status in piscina.js --- src/worker/piscina.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/worker/piscina.js b/src/worker/piscina.js index 7287bfe3..016c90ff 100644 --- a/src/worker/piscina.js +++ b/src/worker/piscina.js @@ -1,6 +1,5 @@ const Sentry = require('@sentry/node') const { isMainThread, threadId } = require('worker_threads') -const { status } = require('../status') if (isMainThread) { const Piscina = require('piscina') @@ -10,7 +9,7 @@ if (isMainThread) { const piscina = new Piscina(createConfig(serverConfig, __filename)) piscina.on('error', (error) => { Sentry.captureException(error) - status.error('โš ๏ธ', 'Piscina worker thread error:\n', error) + console.error('โš ๏ธ', 'Piscina worker thread error:\n', error) }) return piscina }, From 076700abd7d912adbb7a19abb3b399ef8d0828c5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 02:31:23 +0100 Subject: [PATCH 178/218] Revert "Run the vm suite universally" This reverts commit 966056fe8b1479c85cdcd0494df5ce58e50f103a. --- tests/{ => postgres}/vm.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) rename tests/{ => postgres}/vm.test.ts (98%) diff --git a/tests/vm.test.ts b/tests/postgres/vm.test.ts similarity index 98% rename from tests/vm.test.ts rename to tests/postgres/vm.test.ts index 48fc0dc7..8d48cdec 100644 --- a/tests/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -1,12 +1,12 @@ -import { createPluginConfigVM } from '../src/vm' -import { PluginsServer } from '../src/types' +import { createPluginConfigVM } from '../../src/vm' +import { PluginsServer } from '../../src/types' import { PluginEvent } from '@posthog/plugin-scaffold' -import { createServer } from '../src/server' +import { createServer } from '../../src/server' import * as fetch from 'node-fetch' -import { delay } from '../src/utils' -import Client from '../src/celery/client' -import { resetTestDatabase } from './helpers/sql' -import { pluginConfig39 } from './helpers/plugins' +import { delay } from '../../src/utils' +import Client from '../../src/celery/client' +import { resetTestDatabase } from '../helpers/sql' +import { pluginConfig39 } from '../helpers/plugins' jest.mock('../src/celery/client') From e3b83506635064414472e2dda854d3c1436276ce Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 02:31:33 +0100 Subject: [PATCH 179/218] Fix sanitizeSqlIdentifier test --- tests/utils.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 56c5f8e9..4c5aebdd 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -303,6 +303,6 @@ describe('sanitizeSqlIdentifier', () => { const sanitizedIdentifier = sanitizeSqlIdentifier(rawIdentifier) - expect(sanitizeSqlIdentifier).toStrictEqual('"some_field; DROP TABLE actually_an_injection;"') + expect(sanitizedIdentifier).toStrictEqual('"some_field; DROP TABLE actually_an_injection;"') }) }) From 6baed1852f9132e41436866c03b04d1ee7744d9c Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 09:40:15 +0100 Subject: [PATCH 180/218] Use waitForProcessedMessages --- tests/clickhouse/process-event.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 6734cb2a..12e087df 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -44,5 +44,7 @@ test('event is passed through', async () => { now, sent_at: null, }) + const processedMessages = kafkaObserver.waitForProcessedMessages(1) + console.log(processedMessages) expect(1).toEqual(1) }) From 43515af55a7d874764868b2ab081672bfbc4abe5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 09:52:19 +0100 Subject: [PATCH 181/218] Optimize updatePersonProperties --- src/ingestion/process-event.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index b6e8ae6b..15efc617 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -114,12 +114,8 @@ export class EventsProcessor { if (properties['$anon_distinct_id']) { await this.alias(properties['$anon_distinct_id'], distinctId, teamId) } - // TODO: roll the two updatePersonProperties calls into one to `UPDATE posthog_person` only once - if (properties['$set']) { - this.updatePersonProperties(teamId, distinctId, properties['$set']) - } - if (properties['$set_once']) { - this.updatePersonProperties(teamId, distinctId, properties['$set_once'], true) + if (properties['$set'] || properties['$set_once']) { + this.updatePersonProperties(teamId, distinctId, properties['$set'] || {}, properties['$set_once'] || {}) } this.setIsIdentified(teamId, distinctId) } @@ -153,7 +149,7 @@ export class EventsProcessor { teamId: number, distinctId: string, properties: Properties, - setOnce = false + propertiesOnce: Properties ): Promise { let personFound = await this.db.fetchPerson(teamId, distinctId) if (!personFound) { @@ -173,9 +169,7 @@ export class EventsProcessor { personFound = await this.db.fetchPerson(teamId, distinctId) } } - const updatedProperties: Properties = setOnce - ? { ...properties, ...personFound!.properties } - : { ...personFound!.properties, ...properties } + const updatedProperties: Properties = { ...propertiesOnce, ...personFound!.properties, ...properties } return await this.db.updatePerson(personFound!, { properties: updatedProperties }) } From 12af46345d6227a4664bb240dd746fdf4138417a Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 10:24:41 +0100 Subject: [PATCH 182/218] Rework DummyPostHog to capturing internal server events efficiently --- README.md | 1 + src/config.ts | 2 ++ src/extensions/posthog.ts | 25 +++++++++++++++++-------- src/ingestion/process-event.ts | 18 ++++++++++++++++-- src/server.ts | 4 +++- src/types.ts | 3 +++ 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 40f65124..fa44660d 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ There's a multitude of settings you can use to control the plugin server. Use th | KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | | KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | | DISABLE_WEB | whether to disable web server | `true` | +| POSTHOG_TEAM_ID | ID of the PostHog team to capture internal server events to | `null` | | WEB_PORT | port for web server to listen on | `3008` | | WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | | LOG_LEVEL | minimum log level | `LogLevel.Info` | diff --git a/src/config.ts b/src/config.ts index 6fbbec7e..44ce34c6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -25,6 +25,7 @@ export function getDefaultConfig(): PluginsServerConfig { BASE_DIR: '.', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'reload-plugins', DISABLE_WEB: true, + POSTHOG_TEAM_ID: null, WEB_PORT: 3008, WEB_HOSTNAME: '0.0.0.0', WORKER_CONCURRENCY: 0, // use all cores @@ -53,6 +54,7 @@ export function getConfigHelp(): Record { BASE_DIR: 'base path for resolving local plugins', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'Redis channel for reload events', DISABLE_WEB: 'whether to disable web server', + POSTHOG_TEAM_ID: 'ID of the PostHog team to capture internal server events to', WEB_PORT: 'port for web server to listen on', WEB_HOSTNAME: 'hostname for web server to listen on', WORKER_CONCURRENCY: 'number of concurrent worker threads', diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index 51a09b04..57f333d1 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -5,6 +5,7 @@ import { PluginsServer, PluginConfig, RawEventMessage } from 'types' import { version } from '../../package.json' import Client from '../celery/client' import { UUIDT } from '../utils' +import { clientId } from '../server' interface InternalData { distinct_id: string @@ -16,11 +17,19 @@ interface InternalData { } export interface DummyPostHog { - capture(event: string, properties?: Record): void + capture(event: string, properties?: Record, customDistinctId?: string): void } -export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig): DummyPostHog { - const distinctId = pluginConfig.plugin?.name || `plugin-id-${pluginConfig.plugin_id}` +export function createPosthog(server: PluginsServer, pluginConfigOrTeamId: PluginConfig | number): DummyPostHog { + let teamId: number + let inferredDistinctId: string + if (typeof pluginConfigOrTeamId === 'number') { + teamId = pluginConfigOrTeamId + inferredDistinctId = clientId + } else { + teamId = pluginConfigOrTeamId.team_id + inferredDistinctId = pluginConfigOrTeamId.plugin?.name || `plugin-id-${pluginConfigOrTeamId.plugin_id}` + } let sendEvent: (data: InternalData) => Promise @@ -39,7 +48,7 @@ export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig) ip: '', site_url: '', data: JSON.stringify(data), - team_id: pluginConfig.team_id, + team_id: teamId, now: data.timestamp, sent_at: data.timestamp, uuid: data.uuid, @@ -54,17 +63,17 @@ export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig) sendEvent = async (data) => { client.sendTask( 'posthog.tasks.process_event.process_event_with_plugins', - [data.distinct_id, null, null, data, pluginConfig.team_id, data.timestamp, data.timestamp], + [data.distinct_id, null, null, data, teamId, data.timestamp, data.timestamp], {} ) } } return { - capture(event, properties = {}) { + capture(event, properties = {}, customDistinctId) { const { timestamp = DateTime.utc().toISO(), ...otherProperties } = properties const data: InternalData = { - distinct_id: distinctId, + distinct_id: customDistinctId || inferredDistinctId, event, timestamp, properties: { @@ -72,7 +81,7 @@ export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig) $lib_version: version, ...otherProperties, }, - team_id: pluginConfig.team_id, + team_id: teamId, uuid: new UUIDT().toString(), } sendEvent(data) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 15efc617..fcb559b3 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -10,18 +10,23 @@ import { ClickHouse } from 'clickhouse' import { DB } from '../db' import { status } from '../status' import * as Sentry from '@sentry/node' +import { createPosthog, DummyPostHog } from '../extensions/posthog' export class EventsProcessor { pluginsServer: PluginsServer db: DB clickhouse: ClickHouse kafkaProducer: Producer + posthog?: DummyPostHog constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer this.db = pluginsServer.db this.clickhouse = pluginsServer.clickhouse! this.kafkaProducer = pluginsServer.kafkaProducer! + if (pluginsServer.POSTHOG_TEAM_ID) { + this.posthog = createPosthog(this.pluginsServer, parseInt(pluginsServer.POSTHOG_TEAM_ID)) + } } public async processEvent( @@ -345,8 +350,17 @@ export class EventsProcessor { // In _capture we only prefetch a couple of fields in Team to avoid fetching too much data let save = false if (!team.ingested_event) { - // First event for the team captured - // TODO: capture "first team event ingested" + if (this.posthog) { + // First event for the team captured + const organizationMembers = await this.db.postgresQuery( + 'SELECT distinct_id FROM posthog_user JOIN posthog_organizationmembership ON posthog_user.id = posthog_organizationmembership.user_id WHERE organization_id = $1', + [team.organization_id] + ) + const distinctIds: { distinct_id: string }[] = (await organizationMembers).rows + for (const { distinct_id } of distinctIds) { + this.posthog.capture('first team event ingested', { team: team.uuid }, distinct_id) + } + } team.ingested_event = true save = true } diff --git a/src/server.ts b/src/server.ts index 781b2341..f8f5474a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,6 +20,8 @@ import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' +export const clientId = `plugin-server-v${version}-${new UUIDT()}` + export async function createServer( config: Partial = {}, threadId: number | null = null @@ -80,7 +82,7 @@ export async function createServer( }, }) kafka = new Kafka({ - clientId: `plugin-server-v${version}-${new UUIDT()}`, + clientId, brokers: serverConfig.KAFKA_HOSTS.split(','), logLevel: logLevel.NOTHING, ssl: kafkaSsl, diff --git a/src/types.ts b/src/types.ts index 6a49fc31..d285dc78 100644 --- a/src/types.ts +++ b/src/types.ts @@ -38,6 +38,7 @@ export interface PluginsServerConfig extends Record { REDIS_URL: string BASE_DIR: string PLUGINS_RELOAD_PUBSUB_CHANNEL: string + POSTHOG_TEAM_ID: string | null DISABLE_WEB: boolean WEB_PORT: number WEB_HOSTNAME: string @@ -200,6 +201,8 @@ export interface EventMessage extends BaseEventMessage { /** Usable Team model. */ export interface Team { id: number + uuid: string + organization_id: string name: string anonymize_ips: boolean api_token: string From 42124f4600572a6ec4c6e175541cd53925f29efd Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 10:28:02 +0100 Subject: [PATCH 183/218] Fix nested test files not being ran --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 06cfda6d..44391c43 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "types": "dist/src/index.d.ts", "main": "dist/src/index.js", "scripts": { - "test": "jest --runInBand tests/*.test.ts", + "test": "jest --runInBand tests/**/*.test.ts", "test:postgres": "yarn test --testPathIgnorePatterns '.*/clickhouse'", "test:clickhouse": "yarn test --testPathIgnorePatterns '.*/postgres'", "benchmark": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/", From 30f08b6c3591991464d26135501d72cecd1e11b2 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 10:28:50 +0100 Subject: [PATCH 184/218] Run prettier over Markdown --- README.md | 62 ++++++++++++++++++++++++++-------------------------- package.json | 2 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index fa44660d..e3c5aa34 100644 --- a/README.md +++ b/README.md @@ -27,37 +27,37 @@ Let's get you developing the plugin server in no time: There's a multitude of settings you can use to control the plugin server. Use them as environment variables. -| Name | Description | Default value | -| ----------------------------- | ---------------------------------------------------------- | ------------------------------------- | -| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | -| REDIS_URL | Redis store URL | `'redis://localhost'` | -| BASE_DIR | base path for resolving local plugins | `'.'` | -| WORKER_CONCURRENCY | number of concurrent worker threads | `0` โ€“ all cores | -| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | -| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | -| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | -| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | -| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | -| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | -| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | -| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | -| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | -| CLICKHOUSE_CA | ClickHouse CA certs | `null` | -| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | -| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | -| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | -| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | -| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | -| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | -| DISABLE_WEB | whether to disable web server | `true` | -| POSTHOG_TEAM_ID | ID of the PostHog team to capture internal server events to | `null` | -| WEB_PORT | port for web server to listen on | `3008` | -| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | -| LOG_LEVEL | minimum log level | `LogLevel.Info` | -| SENTRY_DSN | Sentry ingestion URL | `null` | -| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | -| STATSD_PORT | StatsD port | `8125` | -| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | +| Name | Description | Default value | +| ----------------------------- | ----------------------------------------------------------- | ------------------------------------- | +| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | +| REDIS_URL | Redis store URL | `'redis://localhost'` | +| BASE_DIR | base path for resolving local plugins | `'.'` | +| WORKER_CONCURRENCY | number of concurrent worker threads | `0` โ€“ all cores | +| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | +| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | +| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | +| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | +| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | +| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | +| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | +| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | +| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | +| CLICKHOUSE_CA | ClickHouse CA certs | `null` | +| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | +| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | +| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | +| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | +| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | +| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | +| DISABLE_WEB | whether to disable web server | `true` | +| POSTHOG_TEAM_ID | ID of the PostHog team to capture internal server events to | `null` | +| WEB_PORT | port for web server to listen on | `3008` | +| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | +| LOG_LEVEL | minimum log level | `LogLevel.Info` | +| SENTRY_DSN | Sentry ingestion URL | `null` | +| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | +| STATSD_PORT | StatsD port | `8125` | +| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | ## Questions? diff --git a/package.json b/package.json index 44391c43..b00e5a3a 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,6 @@ }, "lint-staged": { "*.{js,ts}": "eslint --fix", - "*.{js,ts,css,scss,json,yml}": "prettier --write" + "*.{js,ts,css,scss,json,yml,md}": "prettier --write" } } From 8b4a9051a149de43d194cc674d77c5f51b31905f Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 10:35:24 +0100 Subject: [PATCH 185/218] Fix mock paths --- tests/postgres/vm.test.ts | 2 +- tests/postgres/worker.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index 8d48cdec..7a65b9ee 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -8,7 +8,7 @@ import Client from '../../src/celery/client' import { resetTestDatabase } from '../helpers/sql' import { pluginConfig39 } from '../helpers/plugins' -jest.mock('../src/celery/client') +jest.mock('../../src/celery/client') const defaultEvent = { distinct_id: 'my_id', diff --git a/tests/postgres/worker.test.ts b/tests/postgres/worker.test.ts index a58e4a7f..06355dfa 100644 --- a/tests/postgres/worker.test.ts +++ b/tests/postgres/worker.test.ts @@ -7,7 +7,7 @@ import { makePiscina } from '../../src/worker/piscina' import Client from '../../src/celery/client' import { resetTestDatabase } from '../helpers/sql' -jest.mock('../src/sql') +jest.mock('../../src/sql') jest.setTimeout(600000) // 600 sec timeout function createEvent(index = 0): PluginEvent { From 839ff1eab1f7146ef285c5694e2880db25becbb1 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 10:39:24 +0100 Subject: [PATCH 186/218] Fix issues --- .github/workflows/ci.yml | 2 +- src/config.ts | 2 +- tests/clickhouse/process-event.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efb276c2..bb18c9c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: env: REDIS_URL: 'redis://localhost' CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'test_posthog' + CLICKHOUSE_DATABASE: 'default' KAFKA_ENABLED: 'true' KAFKA_HOSTS: 'localhost:9092' diff --git a/src/config.ts b/src/config.ts index 44ce34c6..84627c71 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,7 +10,7 @@ export function getDefaultConfig(): PluginsServerConfig { CELERY_DEFAULT_QUEUE: 'celery', DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/test_posthog' : 'postgres://localhost:5432/posthog', CLICKHOUSE_HOST: 'localhost', - CLICKHOUSE_DATABASE: isTestEnv ? 'test_posthog' : 'default', + CLICKHOUSE_DATABASE: 'default', CLICKHOUSE_USERNAME: 'default', CLICKHOUSE_PASSWORD: null, CLICKHOUSE_CA: null, diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 12e087df..e66afd6e 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -25,7 +25,7 @@ afterEach(() => { test('event is passed through', async () => { const uuid = new UUIDT().toString() - const now = new DateTime() + const now = DateTime.utc() kafkaObserver.handOffMessage({ distinct_id: 'abcd', ip: '1.1.1.1', From 0de3ebb9383e2c7444ec022820ec4ab54612eac0 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 11:47:10 +0100 Subject: [PATCH 187/218] Fix some --- .github/workflows/ci.yml | 2 +- tests/clickhouse/process-event.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb18c9c9..bbb65a5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: env: REDIS_URL: 'redis://localhost' CLICKHOUSE_HOST: 'localhost' - CLICKHOUSE_DATABASE: 'default' + CLICKHOUSE_DATABASE: 'posthog_test' KAFKA_ENABLED: 'true' KAFKA_HOSTS: 'localhost:9092' diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index e66afd6e..f8349a2f 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -26,7 +26,7 @@ afterEach(() => { test('event is passed through', async () => { const uuid = new UUIDT().toString() const now = DateTime.utc() - kafkaObserver.handOffMessage({ + await kafkaObserver.handOffMessage({ distinct_id: 'abcd', ip: '1.1.1.1', site_url: 'x.com', @@ -44,7 +44,7 @@ test('event is passed through', async () => { now, sent_at: null, }) - const processedMessages = kafkaObserver.waitForProcessedMessages(1) + const processedMessages = await kafkaObserver.waitForProcessedMessages(1) console.log(processedMessages) expect(1).toEqual(1) }) From 25035c856b05014091a309512446f6069b45a2d3 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 12:05:37 +0100 Subject: [PATCH 188/218] Don't truncate Kafka tables --- tests/helpers/clickhouse.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/helpers/clickhouse.ts b/tests/helpers/clickhouse.ts index 936d5e21..aef8f751 100644 --- a/tests/helpers/clickhouse.ts +++ b/tests/helpers/clickhouse.ts @@ -11,10 +11,6 @@ export async function resetTestDatabaseClickhouse(): Promise { }) await clickhouse.query('TRUNCATE events').toPromise() await clickhouse.query('TRUNCATE events_mv').toPromise() - await clickhouse.query('TRUNCATE kafka_events').toPromise() - await clickhouse.query('TRUNCATE kafka_person').toPromise() - await clickhouse.query('TRUNCATE kafka_person_distinct_id').toPromise() - await clickhouse.query('TRUNCATE kafka_session_recording_events').toPromise() await clickhouse.query('TRUNCATE person').toPromise() await clickhouse.query('TRUNCATE person_distinct_id').toPromise() await clickhouse.query('TRUNCATE person_mv').toPromise() From b443240f8d7ab796b02a5a09f9f66123b720c938 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 12:08:03 +0100 Subject: [PATCH 189/218] Start kafkaObserver inside test --- tests/clickhouse/process-event.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index f8349a2f..5f96c49a 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -1,10 +1,7 @@ -import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows, setError } from '../../src/sql' -import { PluginConfig, PluginError, PluginsServer } from '../../src/types' +import { PluginsServer } from '../../src/types' import { createServer } from '../../src/server' import { resetTestDatabase } from '../helpers/sql' -import { commonOrganizationId } from '../helpers/plugins' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' -import { Consumer, EachMessagePayload, Kafka, Producer } from 'kafkajs' import { KafkaObserver } from '../helpers/kafka' import { UUIDT } from '../../src/utils' import { DateTime } from 'luxon' @@ -17,7 +14,6 @@ beforeEach(async () => { ;[server, closeServer] = await createServer() await resetTestDatabase(`const processEvent = event => event`) await resetTestDatabaseClickhouse() - await kafkaObserver.start() }) afterEach(() => { closeServer() @@ -26,6 +22,8 @@ afterEach(() => { test('event is passed through', async () => { const uuid = new UUIDT().toString() const now = DateTime.utc() + await kafkaObserver.start() + await kafkaObserver.handOffMessage({ distinct_id: 'abcd', ip: '1.1.1.1', @@ -45,6 +43,7 @@ test('event is passed through', async () => { sent_at: null, }) const processedMessages = await kafkaObserver.waitForProcessedMessages(1) + console.log(processedMessages) expect(1).toEqual(1) }) From ba2b3bc29da39896013430dd7dd89ba72fd965e9 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 12:15:18 +0100 Subject: [PATCH 190/218] Support Team.is_demo --- tests/helpers/sql.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index f0710b0b..94eaa75d 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -40,6 +40,7 @@ export async function resetTestDatabase(code: string): Promise { session_recording_opt_in: true, plugins_opt_in: true, opt_out_capture: false, + is_demo: false, }) } for (const plugin of mocks.pluginRows) { From 64ba5731dca4534814c5d197570cdf121e4d40f7 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 12:32:42 +0100 Subject: [PATCH 191/218] Increase process-event tests timeout to 60 s --- tests/clickhouse/process-event.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 5f96c49a..38ee1ee1 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -6,6 +6,8 @@ import { KafkaObserver } from '../helpers/kafka' import { UUIDT } from '../../src/utils' import { DateTime } from 'luxon' +jest.setTimeout(60_000) // 60 sec timeout + let server: PluginsServer let closeServer: () => Promise const kafkaObserver = new KafkaObserver() From 816c727264f3c1a21c756f6ea0f7307747b8f566 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 12:56:55 +0100 Subject: [PATCH 192/218] Cache Python location in ClickHouse CI --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbb65a5b..2d23f7b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,11 @@ jobs: with: node-version: 14 + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('posthog/requirements.txt') }} + - name: Install requirements.txt dependencies with pip run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt From 438fa3bb98ebf738aa7398b4759642be126c4bb7 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 12:57:12 +0100 Subject: [PATCH 193/218] Debug KafkaObserver --- tests/helpers/kafka.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 66d0fab0..eb03fa4e 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -28,18 +28,24 @@ export class KafkaObserver extends EventEmitter { } public async start(): Promise { + console.info('observer started!') if (this.isStarted) { return } this.isStarted = true return await new Promise(async (resolve, reject) => { + console.info('connecting producer') await this.producer.connect() + console.info('subscribing consumer') await this.consumer.subscribe({ topic: KAFKA_EVENTS }) + console.info('running consumer') await this.consumer.run({ eachMessage: async (payload) => { + console.info('message received!') this.emit('message', payload) }, }) + console.info('setting group join and crash listeners') const { GROUP_JOIN, CRASH } = this.consumer.events this.consumer.on(GROUP_JOIN, () => resolve()) this.consumer.on(CRASH, ({ payload: { error } }) => reject(error)) @@ -48,13 +54,17 @@ export class KafkaObserver extends EventEmitter { public async stop(): Promise { this.removeAllListeners() + console.info('disconnecting producer') + await this.producer.disconnect() + console.info('stopping consumer') await this.consumer.stop() + console.info('disconnecting consumer') await this.consumer.disconnect() - await this.producer.disconnect() } public async handOffMessage(message: EventMessage): Promise { - this.producer.send({ + console.info('producing message') + await this.producer.send({ topic: KAFKA_EVENTS_INGESTION_HANDOFF, messages: [{ value: Buffer.from(JSON.stringify(message)) }], }) @@ -62,17 +72,20 @@ export class KafkaObserver extends EventEmitter { public async waitForProcessedMessages(numberOfMessages: number): Promise { return await new Promise((resolve, reject) => { + console.info('waiting for processed messages') const accumulator: EventMessage[] = [] const timeoutSeconds = numberOfMessages * 2 // give every message 2 s on average to show up - setTimeout( + const rejectTimeout = setTimeout( () => reject(`Timed out waiting ${timeoutSeconds} seconds for ${numberOfMessages} message(s) from Kafka`), timeoutSeconds * 1000 ) const onMessage = (payload: EachMessagePayload) => { + console.info('message received') accumulator.push(parseRawEventMessage(JSON.parse(payload.message.value!.toString()))) if (accumulator.length >= numberOfMessages) { this.removeListener('message', onMessage) + clearTimeout(rejectTimeout) resolve(accumulator) } } From f9364ad35a6ca037564ed6dcd68c072a603b6ac5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 13:01:45 +0100 Subject: [PATCH 194/218] Cache pip in all CI jobs --- .github/workflows/ci.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d23f7b0..faeda4cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,8 +73,15 @@ jobs: with: node-version: 14 + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('posthog/requirements.txt') }} + - name: Install requirements.txt dependencies with pip - run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt + run: | + pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r posthog/requirements.txt - name: Set up databases env: @@ -167,7 +174,9 @@ jobs: key: ${{ env.pythonLocation }}-${{ hashFiles('posthog/requirements.txt') }} - name: Install requirements.txt dependencies with pip - run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt + run: | + pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r posthog/requirements.txt - name: Set up databases env: @@ -235,8 +244,15 @@ jobs: with: node-version: 14 + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('posthog/requirements.txt') }} + - name: Install requirements.txt dependencies with pip - run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt + run: | + pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r posthog/requirements.txt - name: Set up databases env: From 99afcf528ef2f4b8eb443adccd65376b5492e3d3 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 14:09:46 +0100 Subject: [PATCH 195/218] Add bash script for running tests locally --- run-tests-locally.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 run-tests-locally.sh diff --git a/run-tests-locally.sh b/run-tests-locally.sh new file mode 100755 index 00000000..1839b938 --- /dev/null +++ b/run-tests-locally.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set +ex + +if [[ -z $PRIMARY_DB ]]; then + echo 'You must provide var PRIMARY_DB! Either "postgres" or "clickhouse"' + exit 1 +fi + +export REDIS_URL='redis://localhost' +export CLICKHOUSE_HOST=localhost +export CLICKHOUSE_DATABASE=posthog_test +export KAFKA_ENABLED=true +export KAFKA_HOSTS=kafka:9092 + +source ../posthog/env/bin/activate +SECRET_KEY=abc DATABASE_URL='postgres://postgres:postgres@localhost:5432/posthog' TEST=true python ../posthog/manage.py setup_test_environment +DATABASE_URL='postgres://postgres:postgres@localhost:5432/test_posthog' yarn test:$PRIMARY_DB From f7fe1f498b29c1788a1ecaf05aa4aba949020afc Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 27 Jan 2021 15:04:25 +0100 Subject: [PATCH 196/218] Try out a different way of watching Kafka messages --- tests/clickhouse/process-event.test.ts | 11 +++-- tests/helpers/kafka.ts | 56 ++++++++++++++++---------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 38ee1ee1..3ee2d62f 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -2,11 +2,11 @@ import { PluginsServer } from '../../src/types' import { createServer } from '../../src/server' import { resetTestDatabase } from '../helpers/sql' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' -import { KafkaObserver } from '../helpers/kafka' +import { KafkaCollector, KafkaObserver } from '../helpers/kafka' import { UUIDT } from '../../src/utils' import { DateTime } from 'luxon' -jest.setTimeout(60_000) // 60 sec timeout +jest.setTimeout(180_000) // 3 minute timeout let server: PluginsServer let closeServer: () => Promise @@ -24,8 +24,10 @@ afterEach(() => { test('event is passed through', async () => { const uuid = new UUIDT().toString() const now = DateTime.utc() + console.log('starting kafka observer') await kafkaObserver.start() - + console.log('sending message') + const kafkaCollector = new KafkaCollector(kafkaObserver) await kafkaObserver.handOffMessage({ distinct_id: 'abcd', ip: '1.1.1.1', @@ -44,7 +46,8 @@ test('event is passed through', async () => { now, sent_at: null, }) - const processedMessages = await kafkaObserver.waitForProcessedMessages(1) + console.log('waiting for messages') + const processedMessages = await kafkaCollector.collect(1) console.log(processedMessages) expect(1).toEqual(1) diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index eb03fa4e..569372cf 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -21,8 +21,7 @@ export class KafkaObserver extends EventEmitter { }) this.producer = this.kafka.producer() this.consumer = this.kafka.consumer({ - groupId: 'clickhouse-ingestion', - readUncommitted: false, + groupId: 'clickhouse-ingestion-test', }) this.isStarted = false } @@ -46,8 +45,15 @@ export class KafkaObserver extends EventEmitter { }, }) console.info('setting group join and crash listeners') - const { GROUP_JOIN, CRASH } = this.consumer.events - this.consumer.on(GROUP_JOIN, () => resolve()) + return resolve() + const { CONNECT, GROUP_JOIN, CRASH } = this.consumer.events + this.consumer.on(CONNECT, () => { + console.log('consumer connected to kafka') + }) + this.consumer.on(GROUP_JOIN, () => { + console.log('joined group') + resolve() + }) this.consumer.on(CRASH, ({ payload: { error } }) => reject(error)) }) } @@ -69,27 +75,33 @@ export class KafkaObserver extends EventEmitter { messages: [{ value: Buffer.from(JSON.stringify(message)) }], }) } +} + +export class KafkaCollector extends EventEmitter { + collection: EventMessage[] + kafkaObserver: KafkaObserver + + constructor(kafkaObserver: KafkaObserver) { + super() + this.collection = [] + this.kafkaObserver = kafkaObserver + kafkaObserver.addListener('message', (payload: EachMessagePayload) => { + console.info('message received') + this.collection.push(parseRawEventMessage(JSON.parse(payload.message.value!.toString()))) + this.emit('message') + }) + } - public async waitForProcessedMessages(numberOfMessages: number): Promise { - return await new Promise((resolve, reject) => { - console.info('waiting for processed messages') - const accumulator: EventMessage[] = [] - const timeoutSeconds = numberOfMessages * 2 // give every message 2 s on average to show up - const rejectTimeout = setTimeout( - () => - reject(`Timed out waiting ${timeoutSeconds} seconds for ${numberOfMessages} message(s) from Kafka`), - timeoutSeconds * 1000 - ) - const onMessage = (payload: EachMessagePayload) => { - console.info('message received') - accumulator.push(parseRawEventMessage(JSON.parse(payload.message.value!.toString()))) - if (accumulator.length >= numberOfMessages) { - this.removeListener('message', onMessage) - clearTimeout(rejectTimeout) - resolve(accumulator) + async collect(numberOfMessages: number): Promise { + return await new Promise((resolve) => { + const resolveIfCollectedEnough = () => { + if (this.collection.length >= numberOfMessages) { + this.removeListener('message', resolveIfCollectedEnough) + resolve(this.collection) } } - this.addListener('message', onMessage) + this.addListener('message', resolveIfCollectedEnough) + resolveIfCollectedEnough() }) } } From 644987371d6cb5aa6e0313956b96a5f17d30c249 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Jan 2021 02:59:08 +0100 Subject: [PATCH 197/218] Simplify DummyPostHog back again --- README.md | 61 +++++++++++++++++----------------- src/config.ts | 2 -- src/extensions/posthog.ts | 25 +++++--------- src/ingestion/process-event.ts | 5 --- src/server.ts | 4 +-- src/types.ts | 1 - 6 files changed, 39 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index e3c5aa34..40f65124 100644 --- a/README.md +++ b/README.md @@ -27,37 +27,36 @@ Let's get you developing the plugin server in no time: There's a multitude of settings you can use to control the plugin server. Use them as environment variables. -| Name | Description | Default value | -| ----------------------------- | ----------------------------------------------------------- | ------------------------------------- | -| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | -| REDIS_URL | Redis store URL | `'redis://localhost'` | -| BASE_DIR | base path for resolving local plugins | `'.'` | -| WORKER_CONCURRENCY | number of concurrent worker threads | `0` โ€“ all cores | -| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | -| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | -| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | -| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | -| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | -| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | -| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | -| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | -| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | -| CLICKHOUSE_CA | ClickHouse CA certs | `null` | -| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | -| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | -| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | -| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | -| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | -| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | -| DISABLE_WEB | whether to disable web server | `true` | -| POSTHOG_TEAM_ID | ID of the PostHog team to capture internal server events to | `null` | -| WEB_PORT | port for web server to listen on | `3008` | -| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | -| LOG_LEVEL | minimum log level | `LogLevel.Info` | -| SENTRY_DSN | Sentry ingestion URL | `null` | -| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | -| STATSD_PORT | StatsD port | `8125` | -| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | +| Name | Description | Default value | +| ----------------------------- | ---------------------------------------------------------- | ------------------------------------- | +| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | +| REDIS_URL | Redis store URL | `'redis://localhost'` | +| BASE_DIR | base path for resolving local plugins | `'.'` | +| WORKER_CONCURRENCY | number of concurrent worker threads | `0` โ€“ all cores | +| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | +| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | +| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | +| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | +| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | +| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | +| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | +| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | +| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | +| CLICKHOUSE_CA | ClickHouse CA certs | `null` | +| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | +| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | +| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | +| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | +| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | +| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | +| DISABLE_WEB | whether to disable web server | `true` | +| WEB_PORT | port for web server to listen on | `3008` | +| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | +| LOG_LEVEL | minimum log level | `LogLevel.Info` | +| SENTRY_DSN | Sentry ingestion URL | `null` | +| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | +| STATSD_PORT | StatsD port | `8125` | +| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | ## Questions? diff --git a/src/config.ts b/src/config.ts index 84627c71..1b6c108f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -25,7 +25,6 @@ export function getDefaultConfig(): PluginsServerConfig { BASE_DIR: '.', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'reload-plugins', DISABLE_WEB: true, - POSTHOG_TEAM_ID: null, WEB_PORT: 3008, WEB_HOSTNAME: '0.0.0.0', WORKER_CONCURRENCY: 0, // use all cores @@ -54,7 +53,6 @@ export function getConfigHelp(): Record { BASE_DIR: 'base path for resolving local plugins', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'Redis channel for reload events', DISABLE_WEB: 'whether to disable web server', - POSTHOG_TEAM_ID: 'ID of the PostHog team to capture internal server events to', WEB_PORT: 'port for web server to listen on', WEB_HOSTNAME: 'hostname for web server to listen on', WORKER_CONCURRENCY: 'number of concurrent worker threads', diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index 57f333d1..51a09b04 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -5,7 +5,6 @@ import { PluginsServer, PluginConfig, RawEventMessage } from 'types' import { version } from '../../package.json' import Client from '../celery/client' import { UUIDT } from '../utils' -import { clientId } from '../server' interface InternalData { distinct_id: string @@ -17,19 +16,11 @@ interface InternalData { } export interface DummyPostHog { - capture(event: string, properties?: Record, customDistinctId?: string): void + capture(event: string, properties?: Record): void } -export function createPosthog(server: PluginsServer, pluginConfigOrTeamId: PluginConfig | number): DummyPostHog { - let teamId: number - let inferredDistinctId: string - if (typeof pluginConfigOrTeamId === 'number') { - teamId = pluginConfigOrTeamId - inferredDistinctId = clientId - } else { - teamId = pluginConfigOrTeamId.team_id - inferredDistinctId = pluginConfigOrTeamId.plugin?.name || `plugin-id-${pluginConfigOrTeamId.plugin_id}` - } +export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig): DummyPostHog { + const distinctId = pluginConfig.plugin?.name || `plugin-id-${pluginConfig.plugin_id}` let sendEvent: (data: InternalData) => Promise @@ -48,7 +39,7 @@ export function createPosthog(server: PluginsServer, pluginConfigOrTeamId: Plugi ip: '', site_url: '', data: JSON.stringify(data), - team_id: teamId, + team_id: pluginConfig.team_id, now: data.timestamp, sent_at: data.timestamp, uuid: data.uuid, @@ -63,17 +54,17 @@ export function createPosthog(server: PluginsServer, pluginConfigOrTeamId: Plugi sendEvent = async (data) => { client.sendTask( 'posthog.tasks.process_event.process_event_with_plugins', - [data.distinct_id, null, null, data, teamId, data.timestamp, data.timestamp], + [data.distinct_id, null, null, data, pluginConfig.team_id, data.timestamp, data.timestamp], {} ) } } return { - capture(event, properties = {}, customDistinctId) { + capture(event, properties = {}) { const { timestamp = DateTime.utc().toISO(), ...otherProperties } = properties const data: InternalData = { - distinct_id: customDistinctId || inferredDistinctId, + distinct_id: distinctId, event, timestamp, properties: { @@ -81,7 +72,7 @@ export function createPosthog(server: PluginsServer, pluginConfigOrTeamId: Plugi $lib_version: version, ...otherProperties, }, - team_id: teamId, + team_id: pluginConfig.team_id, uuid: new UUIDT().toString(), } sendEvent(data) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index fcb559b3..cf1b34a3 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -10,23 +10,18 @@ import { ClickHouse } from 'clickhouse' import { DB } from '../db' import { status } from '../status' import * as Sentry from '@sentry/node' -import { createPosthog, DummyPostHog } from '../extensions/posthog' export class EventsProcessor { pluginsServer: PluginsServer db: DB clickhouse: ClickHouse kafkaProducer: Producer - posthog?: DummyPostHog constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer this.db = pluginsServer.db this.clickhouse = pluginsServer.clickhouse! this.kafkaProducer = pluginsServer.kafkaProducer! - if (pluginsServer.POSTHOG_TEAM_ID) { - this.posthog = createPosthog(this.pluginsServer, parseInt(pluginsServer.POSTHOG_TEAM_ID)) - } } public async processEvent( diff --git a/src/server.ts b/src/server.ts index f8f5474a..781b2341 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,8 +20,6 @@ import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' -export const clientId = `plugin-server-v${version}-${new UUIDT()}` - export async function createServer( config: Partial = {}, threadId: number | null = null @@ -82,7 +80,7 @@ export async function createServer( }, }) kafka = new Kafka({ - clientId, + clientId: `plugin-server-v${version}-${new UUIDT()}`, brokers: serverConfig.KAFKA_HOSTS.split(','), logLevel: logLevel.NOTHING, ssl: kafkaSsl, diff --git a/src/types.ts b/src/types.ts index d285dc78..221dc21d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -38,7 +38,6 @@ export interface PluginsServerConfig extends Record { REDIS_URL: string BASE_DIR: string PLUGINS_RELOAD_PUBSUB_CHANNEL: string - POSTHOG_TEAM_ID: string | null DISABLE_WEB: boolean WEB_PORT: number WEB_HOSTNAME: string From 68ce77e0c0ff8089260c3ac3df9338464e2bf8f3 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Jan 2021 03:11:47 +0100 Subject: [PATCH 198/218] Use posthog-js-lite in EventsProcessor --- package.json | 1 + src/ingestion/process-event.ts | 22 ++++++++++++---------- yarn.lock | 5 +++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index b00e5a3a..a76030d2 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "node-schedule": "^1.3.2", "pg": "^8.4.2", "piscina": "^2.1.0", + "posthog-js-lite": "^0.0.5", "protobufjs": "^6.10.2", "redlock": "^4.2.0", "tar-stream": "^2.1.4", diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index cf1b34a3..790fa862 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -10,18 +10,21 @@ import { ClickHouse } from 'clickhouse' import { DB } from '../db' import { status } from '../status' import * as Sentry from '@sentry/node' +import { nodePostHog } from 'posthog-js-lite/dist/src/targets/node' export class EventsProcessor { pluginsServer: PluginsServer db: DB clickhouse: ClickHouse kafkaProducer: Producer + posthog: ReturnType constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer this.db = pluginsServer.db this.clickhouse = pluginsServer.clickhouse! this.kafkaProducer = pluginsServer.kafkaProducer! + this.posthog = nodePostHog('sTMFPsFhdP1Ssg') } public async processEvent( @@ -345,16 +348,15 @@ export class EventsProcessor { // In _capture we only prefetch a couple of fields in Team to avoid fetching too much data let save = false if (!team.ingested_event) { - if (this.posthog) { - // First event for the team captured - const organizationMembers = await this.db.postgresQuery( - 'SELECT distinct_id FROM posthog_user JOIN posthog_organizationmembership ON posthog_user.id = posthog_organizationmembership.user_id WHERE organization_id = $1', - [team.organization_id] - ) - const distinctIds: { distinct_id: string }[] = (await organizationMembers).rows - for (const { distinct_id } of distinctIds) { - this.posthog.capture('first team event ingested', { team: team.uuid }, distinct_id) - } + // First event for the team captured + const organizationMembers = await this.db.postgresQuery( + 'SELECT distinct_id FROM posthog_user JOIN posthog_organizationmembership ON posthog_user.id = posthog_organizationmembership.user_id WHERE organization_id = $1', + [team.organization_id] + ) + const distinctIds: { distinct_id: string }[] = (await organizationMembers).rows + for (const { distinct_id } of distinctIds) { + this.posthog.identify(distinct_id) + this.posthog.capture('first team event ingested', { team: team.uuid }) } team.ingested_event = true save = true diff --git a/yarn.lock b/yarn.lock index 18bc78fd..a5470402 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5800,6 +5800,11 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" +posthog-js-lite@^0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.5.tgz#984a619190d1c4ef003cb81194d0a6276491d2e8" + integrity sha512-xHVZ9qbBdoqK7G1JVA4k6nheWu50aiEwjJIPQw7Zhi705aX5wr2W8zxdmvtadeEE5qbJNg+/uZ7renOQoqcbJw== + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" From f12419c7447a673f7a2bb4bf77c169bae0b18080 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Jan 2021 03:18:26 +0100 Subject: [PATCH 199/218] Sanitize SQL identifiers more strictly Remove all characters in identifiers that are neither letter, digit or underscore. --- src/utils.ts | 2 +- tests/utils.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 369c5836..e91ac52a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -297,5 +297,5 @@ export function delay(ms: number): Promise { /** Remove all quotes from the provided identifier to prevent SQL injection. */ export function sanitizeSqlIdentifier(unquotedIdentifier: string): string { - return '"' + unquotedIdentifier.replace(/["']/g, '') + '"' + return '"' + unquotedIdentifier.replace(/[^\w\d_]+/g, '') + '"' } diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 4c5aebdd..0331b53a 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -298,11 +298,11 @@ describe('UUIDT', () => { }) describe('sanitizeSqlIdentifier', () => { - it('removes quotes inside and adds quotes around identifier', () => { - const rawIdentifier = 'some_field"; DROP TABLE actually_an_injection;' + it('removes all characters that are neither letter, digit or underscore and adds quotes around identifier', () => { + const rawIdentifier = 'some_field"; DROP TABLE actually_an_injection-9;' const sanitizedIdentifier = sanitizeSqlIdentifier(rawIdentifier) - expect(sanitizedIdentifier).toStrictEqual('"some_field; DROP TABLE actually_an_injection;"') + expect(sanitizedIdentifier).toStrictEqual('"some_fieldDROPTABLEactually_an_injection9"') }) }) From a662dd7a0b13d62cb8526ebc33ebc7016eb79046 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Jan 2021 11:30:26 +0100 Subject: [PATCH 200/218] Don't resolve KafkaObserver start before connection --- tests/helpers/kafka.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 569372cf..37b7dbe7 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -45,7 +45,6 @@ export class KafkaObserver extends EventEmitter { }, }) console.info('setting group join and crash listeners') - return resolve() const { CONNECT, GROUP_JOIN, CRASH } = this.consumer.events this.consumer.on(CONNECT, () => { console.log('consumer connected to kafka') @@ -95,6 +94,7 @@ export class KafkaCollector extends EventEmitter { async collect(numberOfMessages: number): Promise { return await new Promise((resolve) => { const resolveIfCollectedEnough = () => { + console.log('collection:', this.collection) if (this.collection.length >= numberOfMessages) { this.removeListener('message', resolveIfCollectedEnough) resolve(this.collection) From 9c60fa9f6bedaafd411bad6f167033c5f5ab22a5 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Jan 2021 11:30:48 +0100 Subject: [PATCH 201/218] Add ee.tasks.webhooks_ee.post_event_to_webhook_ee --- src/db.ts | 10 +++++++++- src/ingestion/process-event.ts | 20 ++++++++++++++++++-- src/types.ts | 8 ++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/db.ts b/src/db.ts index 3659c7be..5a64697b 100644 --- a/src/db.ts +++ b/src/db.ts @@ -5,7 +5,7 @@ import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { unparsePersonPartial } from './ingestion/utils' -import { Person, PersonDistinctId, RawPerson } from './types' +import { Person, PersonDistinctId, RawPerson, RawOrganization } from './types' import { castTimestampOrNow, sanitizeSqlIdentifier } from './utils' /** The recommended way of accessing the database. */ @@ -145,4 +145,12 @@ export class DB { }) } } + + public async fetchOrganization(organizationId: string): Promise { + const selectResult = await this.postgresQuery(`SELECT * FROM posthog_organization WHERE id $1`, [ + organizationId, + ]) + const rawOrganization: RawOrganization = selectResult.rows[0] + return rawOrganization + } } diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 790fa862..decdb1f4 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -11,12 +11,14 @@ import { DB } from '../db' import { status } from '../status' import * as Sentry from '@sentry/node' import { nodePostHog } from 'posthog-js-lite/dist/src/targets/node' +import Client from '../celery/client' export class EventsProcessor { pluginsServer: PluginsServer db: DB clickhouse: ClickHouse kafkaProducer: Producer + celery: Client posthog: ReturnType constructor(pluginsServer: PluginsServer) { @@ -24,6 +26,7 @@ export class EventsProcessor { this.db = pluginsServer.db this.clickhouse = pluginsServer.clickhouse! this.kafkaProducer = pluginsServer.kafkaProducer! + this.celery = new Client(pluginsServer.redis) this.posthog = nodePostHog('sTMFPsFhdP1Ssg') } @@ -341,7 +344,7 @@ export class EventsProcessor { } catch {} } - return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList) + return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList, siteUrl) } private async storeNamesAndProperties(team: Team, event: string, properties: Properties): Promise { @@ -407,7 +410,8 @@ export class EventsProcessor { distinctId: string, properties?: Properties, timestamp?: DateTime | string, - elements?: Element[] + elements?: Element[], + siteUrl?: string ): Promise { const timestampString = castTimestampOrNow(timestamp) const elementsChain = elements && elements.length ? elementsToString(elements) : '' @@ -428,6 +432,18 @@ export class EventsProcessor { messages: [{ key: uuid, value: EventProto.encodeDelimited(EventProto.create(data)).finish() as Buffer }], }) + this.celery.sendTask('ee.tasks.webhooks_ee.post_event_to_webhook_ee', [ + { + event, + properties, + distinct_id: distinctId, + timestamp, + elements_list: elements, + }, + team.id, + siteUrl, + ]) + return data } diff --git a/src/types.ts b/src/types.ts index 221dc21d..067feeec 100644 --- a/src/types.ts +++ b/src/types.ts @@ -197,6 +197,14 @@ export interface EventMessage extends BaseEventMessage { sent_at: DateTime | null } +/** Raw Organization row from database. */ +export interface RawOrganization { + id: string + name: string + created_at: string + updated_at: string +} + /** Usable Team model. */ export interface Team { id: number From 52dc915bc1251e400a414b8fae82f5eb78dcca42 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 1 Feb 2021 09:32:47 +0100 Subject: [PATCH 202/218] Expect Client to have been called twice --- tests/postgres/vm.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index 7a65b9ee..cff117ed 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -712,7 +712,7 @@ test('posthog in runEvery', async () => { const response = await vm.tasks.runEveryMinute.exec() expect(response).toBe('haha') - expect(Client).toHaveBeenCalledTimes(1) + expect(Client).toHaveBeenCalledTimes(2) expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) const mockClientInstance = (Client as any).mock.instances[0] @@ -750,7 +750,7 @@ test('posthog in runEvery with timestamp', async () => { const response = await vm.tasks.runEveryMinute.exec() expect(response).toBe('haha') - expect(Client).toHaveBeenCalledTimes(1) + expect(Client).toHaveBeenCalledTimes(2) expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) const mockClientInstance = (Client as any).mock.instances[0] From 5b291b7b508f38da950e6490b9340b5f104cf999 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 1 Feb 2021 09:44:18 +0100 Subject: [PATCH 203/218] Adjust resetTestDatabase for Organization.personalization --- tests/helpers/sql.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 94eaa75d..6e4463e0 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -19,6 +19,7 @@ export async function resetTestDatabase(code: string): Promise { name: 'TEST ORG', created_at: new Date().toISOString(), updated_at: new Date().toISOString(), + personalization: '{}', }) for (const teamId of teamIds) { await insertRow(db, 'posthog_team', { From 9856cd4a70ab32a243ef1a417a469e672298ba31 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 1 Feb 2021 09:51:13 +0100 Subject: [PATCH 204/218] Update Cliient call asserts --- tests/postgres/vm.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index cff117ed..ef459fc1 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -713,7 +713,8 @@ test('posthog in runEvery', async () => { expect(response).toBe('haha') expect(Client).toHaveBeenCalledTimes(2) - expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) + expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin const mockClientInstance = (Client as any).mock.instances[0] const mockSendTask = mockClientInstance.sendTask @@ -751,7 +752,8 @@ test('posthog in runEvery with timestamp', async () => { expect(response).toBe('haha') expect(Client).toHaveBeenCalledTimes(2) - expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) + expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin const mockClientInstance = (Client as any).mock.instances[0] const mockSendTask = mockClientInstance.sendTask From 3090e4a4c9598d3087d58241792e969da4609ba6 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 11:00:08 +0100 Subject: [PATCH 205/218] Postgres ingestion + process event tests (#116) * start with the postgres event ingestion process event tests * get first test to work * remove siteUrl * pass partial event to test that it still works and retain parity with the django test * another test * refactor * add more tests * opt out of posthog in test mode * add first alias test * always use UTC times when talking to postgres * prevent a crash * bit of clarity to help debug * fix bug with table name * fix bug with passing object instead of id * add some alias tests (all green now) * save merged properties * few more tests * more missing tests * fix test * team event properties test * fix bug * another test (partial) * clarify postgres magic * different timestamp format for creating event in clickhouse & postgresql * make element tests fail * capture first team event test * insert session recording events * generate element hashes * create elements and element groups * "key in object" only works with objects, not arrays * test an extra thing * add missing awaits that caused things to be done out of order * few extra tests * another test * await for things to happen * fix to work with latest master * client is called twice - it's initialized for sending celery tasks on webhooks as well * check webhook celery client queue * split into postgres & shared process event test * add query counter * clickhouse process event tests v0.1 * fix vm test * fix "Your test suite must contain at least one test." error for shared tests * also run non-ingestion tests under test:postgres * Clean up utils.ts * Clean up TimestampFormat * Put get* type Postgres functions in DB class Co-authored-by: Michael Matloka --- package.json | 8 +- src/db.ts | 70 +- src/ingestion/process-event.ts | 161 +++- src/ingestion/utils.ts | 42 + src/server.ts | 16 +- src/types.ts | 29 + src/utils.ts | 16 +- tests/clickhouse/process-event.test.ts | 96 ++- tests/helpers/clickhouse.ts | 8 +- tests/helpers/kafka.ts | 8 +- tests/helpers/plugins.ts | 2 + tests/helpers/sql.ts | 107 ++- tests/postgres/process-event.test.ts | 14 + tests/postgres/vm.test.ts | 8 +- tests/shared/process-event.ts | 1045 ++++++++++++++++++++++++ 15 files changed, 1502 insertions(+), 128 deletions(-) create mode 100644 tests/postgres/process-event.test.ts create mode 100644 tests/shared/process-event.ts diff --git a/package.json b/package.json index a76030d2..bed5959b 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "main": "dist/src/index.js", "scripts": { "test": "jest --runInBand tests/**/*.test.ts", - "test:postgres": "yarn test --testPathIgnorePatterns '.*/clickhouse'", - "test:clickhouse": "yarn test --testPathIgnorePatterns '.*/postgres'", + "test:postgres": "jest --runInBand tests/postgres/*.test.ts tests/*.test.ts", + "test:clickhouse": "jest --runInBand tests/clickhouse/*.test.ts", "benchmark": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/", "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", @@ -23,7 +23,9 @@ "prettier": "prettier --write .", "prettier:check": "prettier --check .", "prepare": "yarn compile:protobuf", - "prepublishOnly": "yarn build" + "prepublishOnly": "yarn build", + "setup:dev": "cd ../posthog && (dropdb test_posthog || echo 'no db to drop') && createdb test_posthog && source env/bin/activate && DATABASE_URL=postgres://localhost:5432/test_posthog DEBUG=1 python manage.py migrate", + "setup:dev:ee": "export DEBUG=1 PRIMARY_DB=clickhouse DATABASE_URL=postgres://posthog:posthog@localhost:5439/test_posthog PGPASSWORD=posthog && cd ../posthog && (dropdb -p 5439 -h localhost -U posthog test_posthog || echo 'no db to drop') && createdb -p 5439 -h localhost -U posthog test_posthog && source env/bin/activate && python manage.py migrate && python manage.py migrate_clickhouse" }, "bin": { "posthog-plugin-server": "bin/posthog-plugin-server" diff --git a/src/db.ts b/src/db.ts index 5a64697b..7b4f430c 100644 --- a/src/db.ts +++ b/src/db.ts @@ -5,8 +5,16 @@ import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { unparsePersonPartial } from './ingestion/utils' -import { Person, PersonDistinctId, RawPerson, RawOrganization } from './types' -import { castTimestampOrNow, sanitizeSqlIdentifier } from './utils' +import { + Person, + PersonDistinctId, + RawPerson, + RawOrganization, + Team, + PostgresSessionRecordingEvent, + Event, +} from './types' +import { castTimestampOrNow, sanitizeSqlIdentifier, UUIDT } from './utils' /** The recommended way of accessing the database. */ export class DB { @@ -30,6 +38,13 @@ export class DB { return this.postgres.query(queryTextOrConfig, values) } + // Person + + public async fetchPersons(): Promise { + const result = await this.postgresQuery('SELECT * FROM posthog_person') + return result.rows as Person[] + } + public async fetchPerson(teamId: number, distinctId: string): Promise { const selectResult = await this.postgresQuery( `SELECT @@ -45,8 +60,10 @@ export class DB { AND posthog_persondistinctid.distinct_id = $2`, [teamId, distinctId] ) - const rawPerson: RawPerson = selectResult.rows[0] - return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } + if (selectResult.rows.length > 0) { + const rawPerson: RawPerson = selectResult.rows[0] + return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } + } } public async createPerson( @@ -55,7 +72,8 @@ export class DB { teamId: number, isUserId: number | null, isIdentified: boolean, - uuid: string + uuid: string, + distinctIds?: string[] ): Promise { const insertResult = await this.postgresQuery( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', @@ -75,16 +93,22 @@ export class DB { messages: [{ value: Buffer.from(JSON.stringify(data)) }], }) } + + for (const distinctId of distinctIds || []) { + await this.addDistinctId(personCreated, distinctId) + } + return personCreated } public async updatePerson(person: Person, update: Partial): Promise { const updatedPerson: Person = { ...person, ...update } + const values = [...Object.values(unparsePersonPartial(update)), person.id] await this.postgresQuery( `UPDATE posthog_person SET ${Object.keys(update).map( (field, index) => sanitizeSqlIdentifier(field) + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, - [...Object.values(unparsePersonPartial(update)), person.id] + values ) if (this.kafkaProducer) { const data = { @@ -103,7 +127,7 @@ export class DB { } public async deletePerson(personId: number): Promise { - await this.postgresQuery('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) + await this.postgresQuery('DELETE FROM posthog_persondistinctid WHERE person_id = $1', [personId]) await this.postgresQuery('DELETE FROM posthog_person WHERE id = $1', [personId]) if (this.clickhouse) { await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() @@ -113,6 +137,16 @@ export class DB { } } + // PersonDistinctId + + public async fetchDistinctIdValues(person: Person): Promise { + const result = await this.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', + [person.id, person.team_id] + ) + return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) + } + public async addDistinctId(person: Person, distinctId: string): Promise { const insertResult = await this.postgresQuery( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', @@ -146,6 +180,8 @@ export class DB { } } + // Organization + public async fetchOrganization(organizationId: string): Promise { const selectResult = await this.postgresQuery(`SELECT * FROM posthog_organization WHERE id $1`, [ organizationId, @@ -153,4 +189,24 @@ export class DB { const rawOrganization: RawOrganization = selectResult.rows[0] return rawOrganization } + + // Event + + public async fetchEvents(): Promise { + const result = await this.postgresQuery('SELECT * FROM posthog_event') + return result.rows as Event[] + } + + // SessionRecordingEvent + + public async fetchSessionRecordingEvents(): Promise { + const result = await this.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') + return result.rows as PostgresSessionRecordingEvent[] + } + + // Element + + public async fetchElements(): Promise { + return (await this.postgresQuery('SELECT * FROM posthog_element')).rows + } } diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index decdb1f4..73a8e0db 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,11 +1,22 @@ import { PluginEvent, Properties } from '@posthog/plugin-scaffold' import { DateTime, Duration } from 'luxon' -import { PluginsServer, Element, Team, Person, PersonDistinctId, CohortPeople, SessionRecordingEvent } from '../types' +import { + CohortPeople, + Element, + ElementGroup, + Person, + PersonDistinctId, + PluginsServer, + PostgresSessionRecordingEvent, + SessionRecordingEvent, + Team, + TimestampFormat, +} from '../types' import { castTimestampOrNow, UUIDT } from '../utils' import { Event as EventProto, IEvent } from '../idl/protos' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' -import { sanitizeEventName, elementsToString } from './utils' +import { elementsToString, hashElements, sanitizeEventName } from './utils' import { ClickHouse } from 'clickhouse' import { DB } from '../db' import { status } from '../status' @@ -26,8 +37,11 @@ export class EventsProcessor { this.db = pluginsServer.db this.clickhouse = pluginsServer.clickhouse! this.kafkaProducer = pluginsServer.kafkaProducer! - this.celery = new Client(pluginsServer.redis) + this.celery = new Client(pluginsServer.redis, pluginsServer.CELERY_DEFAULT_QUEUE) this.posthog = nodePostHog('sTMFPsFhdP1Ssg') + if (process.env.NODE_ENV === 'test') { + this.posthog.optOut() + } } public async processEvent( @@ -53,7 +67,7 @@ export class EventsProcessor { const personUuid = new UUIDT().toString() const ts = this.handleTimestamp(data, now, sentAt) - this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) + await this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) let result: IEvent | SessionRecordingEvent @@ -121,9 +135,14 @@ export class EventsProcessor { await this.alias(properties['$anon_distinct_id'], distinctId, teamId) } if (properties['$set'] || properties['$set_once']) { - this.updatePersonProperties(teamId, distinctId, properties['$set'] || {}, properties['$set_once'] || {}) + await this.updatePersonProperties( + teamId, + distinctId, + properties['$set'] || {}, + properties['$set_once'] || {} + ) } - this.setIsIdentified(teamId, distinctId) + await this.setIsIdentified(teamId, distinctId) } } @@ -139,7 +158,7 @@ export class EventsProcessor { true, new UUIDT().toString() ) - this.db.addDistinctId(personCreated, distinctId) + await this.db.addDistinctId(personCreated, distinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person @@ -190,13 +209,13 @@ export class EventsProcessor { if (oldPerson && !newPerson) { try { - this.db.addDistinctId(oldPerson, distinctId) + await this.db.addDistinctId(oldPerson, distinctId) // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error if (retryIfFailed) { // run everything again to merge the users if needed - this.alias(previousDistinctId, distinctId, teamId, false) + await this.alias(previousDistinctId, distinctId, teamId, false) } } return @@ -204,13 +223,13 @@ export class EventsProcessor { if (!oldPerson && newPerson) { try { - this.db.addDistinctId(newPerson, previousDistinctId) + await this.db.addDistinctId(newPerson, previousDistinctId) // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error if (retryIfFailed) { // run everything again to merge the users if needed - this.alias(previousDistinctId, distinctId, teamId, false) + await this.alias(previousDistinctId, distinctId, teamId, false) } } return @@ -226,21 +245,21 @@ export class EventsProcessor { false, new UUIDT().toString() ) - this.db.addDistinctId(personCreated, distinctId) - this.db.addDistinctId(personCreated, previousDistinctId) + await this.db.addDistinctId(personCreated, distinctId) + await this.db.addDistinctId(personCreated, previousDistinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person if (retryIfFailed) { // Try once more, probably one of the two persons exists now - this.alias(previousDistinctId, distinctId, teamId, false) + await this.alias(previousDistinctId, distinctId, teamId, false) } } return } if (oldPerson && newPerson && oldPerson.id !== newPerson.id) { - this.mergePeople(newPerson, [oldPerson]) + await this.mergePeople(newPerson, [oldPerson]) } } @@ -256,14 +275,14 @@ export class EventsProcessor { } } - await this.db.updatePerson(mergeInto, { created_at: firstSeen }) + await this.db.updatePerson(mergeInto, { created_at: firstSeen, properties: mergeInto.properties }) // merge the distinct_ids for (const otherPerson of peopleToMerge) { const otherPersonDistinctIds: PersonDistinctId[] = ( await this.db.postgresQuery( 'SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', - [otherPerson, mergeInto.team_id] + [otherPerson.id, mergeInto.team_id] ) ).rows for (const personDistinctId of otherPersonDistinctIds) { @@ -321,7 +340,7 @@ export class EventsProcessor { properties['$ip'] = ip } - this.storeNamesAndProperties(team, event, properties) + await this.storeNamesAndProperties(team, event, properties) const pdiSelectResult = await this.db.postgresQuery( 'SELECT COUNT(*) AS pdicount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', @@ -364,13 +383,13 @@ export class EventsProcessor { team.ingested_event = true save = true } - if (team.event_names && !(event in team.event_names)) { + if (team.event_names && !team.event_names.includes(event)) { save = true team.event_names.push(event) team.event_names_with_usage.push({ event: event, usage_count: null, volume: null }) } for (const [key, value] of Object.entries(properties)) { - if (team.event_properties && !(key in team.event_properties)) { + if (team.event_properties && !team.event_properties.includes(key)) { team.event_properties.push(key) team.event_properties_with_usage.push({ key: key, usage_count: null, volume: null }) save = true @@ -378,7 +397,7 @@ export class EventsProcessor { if ( typeof value === 'number' && team.event_properties_numerical && - !(key in team.event_properties_numerical) + !team.event_properties_numerical.includes(key) ) { team.event_properties_numerical.push(key) save = true @@ -395,7 +414,7 @@ export class EventsProcessor { JSON.stringify(team.event_names), JSON.stringify(team.event_names_with_usage), JSON.stringify(team.event_properties), - JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties_with_usage), JSON.stringify(team.event_properties_numerical), team.id, ] @@ -413,7 +432,10 @@ export class EventsProcessor { elements?: Element[], siteUrl?: string ): Promise { - const timestampString = castTimestampOrNow(timestamp) + const timestampString = castTimestampOrNow( + timestamp, + this.kafkaProducer ? TimestampFormat.ClickHouse : TimestampFormat.ISO + ) const elementsChain = elements && elements.length ? elementsToString(elements) : '' const data: IEvent = { @@ -427,10 +449,36 @@ export class EventsProcessor { createdAt: timestampString, } - await this.kafkaProducer.send({ - topic: KAFKA_EVENTS, - messages: [{ key: uuid, value: EventProto.encodeDelimited(EventProto.create(data)).finish() as Buffer }], - }) + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_EVENTS, + messages: [ + { + key: uuid, + value: EventProto.encodeDelimited(EventProto.create(data)).finish() as Buffer, + }, + ], + }) + } else { + let elementsHash = '' + if (elements && elements.length > 0) { + elementsHash = await this.createElementGroup(elements, team.id) + } + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_event (created_at, event, distinct_id, properties, team_id, timestamp, elements, elements_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *', + [ + data.createdAt, + data.event, + distinctId, + data.properties, + data.teamId, + data.timestamp, + JSON.stringify(elements || []), + elementsHash, + ] + ) + const eventCreated = insertResult.rows[0] as Event + } this.celery.sendTask('ee.tasks.webhooks_ee.post_event_to_webhook_ee', [ { @@ -447,6 +495,45 @@ export class EventsProcessor { return data } + private async createElementGroup(elements: Element[], teamId: number): Promise { + const cleanedElements = elements.map((element, index) => ({ ...element, order: index })) + const hash = hashElements(cleanedElements) + + try { + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_elementgroup (hash, team_id) VALUES ($1, $2) RETURNING *', + [hash, teamId] + ) + const elementGroup = insertResult.rows[0] as ElementGroup + for (const element of cleanedElements) { + await this.db.postgresQuery( + 'INSERT INTO posthog_element (text, tag_name, href, attr_id, nth_child, nth_of_type, attributes, "order", event_id, attr_class, group_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)', + [ + element.text, + element.tag_name, + element.href, + element.attr_id, + element.nth_child, + element.nth_of_type, + element.attributes, + element.order, + element.event_id, + element.attr_class, + elementGroup.id, + ] + ) + } + } catch (error) { + // Throw further if not postgres error nr "23505" == "unique_violation" + // https://www.postgresql.org/docs/12/errcodes-appendix.html + if (error.code !== '23505') { + throw error + } + } + + return hash + } + private async createSessionRecordingEvent( uuid: string, team_id: number, @@ -454,7 +541,7 @@ export class EventsProcessor { session_id: string, timestamp: DateTime | string, snapshot_data: Record - ): Promise { + ): Promise { const timestampString = castTimestampOrNow(timestamp) const data: SessionRecordingEvent = { @@ -467,11 +554,19 @@ export class EventsProcessor { created_at: timestampString, } - await this.kafkaProducer.send({ - topic: KAFKA_SESSION_RECORDING_EVENTS, - messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }], - }) - + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_SESSION_RECORDING_EVENTS, + messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }], + }) + } else { + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_sessionrecordingevent (created_at, team_id, distinct_id, session_id, timestamp, snapshot_data) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', + [data.created_at, data.team_id, data.distinct_id, data.session_id, data.timestamp, data.snapshot_data] + ) + const eventCreated = insertResult.rows[0] as PostgresSessionRecordingEvent + return eventCreated + } return data } } diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 8098cd1a..fdab780b 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -1,5 +1,6 @@ import { DateTime } from 'luxon' import { Element, BaseEventMessage, RawEventMessage, EventMessage, BasePerson, RawPerson, Person } from '../types' +import crypto from 'crypto' export function parseRawEventMessage(message: RawEventMessage): EventMessage { return { @@ -67,3 +68,44 @@ export function sanitizeEventName(eventName: any): string { } return eventName.substr(0, 200) } + +/** Escape UTF-8 characters into `\u1234`. */ +function jsonEscapeUtf8(s: string): string { + return s.replace(/[^\x20-\x7F]/g, (x) => '\\u' + ('000' + x.codePointAt(0)?.toString(16)).slice(-4)) +} + +/** Produce output compatible with that of Python's `json.dumps`. */ +function jsonDumps(obj: any): string { + if (typeof obj === 'object' && obj !== null) { + if (Array.isArray(obj)) { + return `[${obj.map(jsonDumps).join(', ')}]` // space after comma + } else { + return `{${Object.keys(obj) // no space after '{' or before '}' + .sort() // must sort the keys of the object! + .map((k) => `${jsonDumps(k)}: ${jsonDumps(obj[k])}`) // space after ':' + .join(', ')}}` // space after ',' + } + } else if (typeof obj === 'string') { + return jsonEscapeUtf8(JSON.stringify(obj)) + } else { + return JSON.stringify(obj) + } +} + +export function hashElements(elements: Element[]): string { + const elementsList = elements.map((element) => ({ + attributes: element.attributes ?? null, + text: element.text ?? null, + tag_name: element.tag_name ?? null, + href: element.href ?? null, + attr_id: element.attr_id ?? null, + attr_class: element.attr_class ?? null, + nth_child: element.nth_child ?? null, + nth_of_type: element.nth_of_type ?? null, + order: element.order ?? null, + })) + + const serializedString = jsonDumps(elementsList) + + return crypto.createHash('md5').update(serializedString).digest('hex') +} diff --git a/src/server.ts b/src/server.ts index 781b2341..8deedd40 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,4 +1,4 @@ -import { Pool } from 'pg' +import { Pool, types as pgTypes } from 'pg' import * as schedule from 'node-schedule' import Redis from 'ioredis' import { Kafka, logLevel, Producer } from 'kafkajs' @@ -19,6 +19,7 @@ import { status } from './status' import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' +import { DateTime } from 'luxon' export async function createServer( config: Partial = {}, @@ -88,6 +89,19 @@ export async function createServer( kafkaProducer = kafka.producer() } + // `node-postgres` will return dates as plain JS Date objects, which will use the local timezone. + // This converts all date fields to a proper luxon UTC DateTime + // Unfortunately this must be done on a global object before initializing the `Pool` + pgTypes.setTypeParser(1083 /* types.TypeId.TIME */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + pgTypes.setTypeParser(1114 /* types.TypeId.TIMESTAMP */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + pgTypes.setTypeParser(1184 /* types.TypeId.TIMESTAMPTZ */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + const postgres = new Pool({ connectionString: serverConfig.DATABASE_URL, ssl: process.env.DEPLOYMENT?.startsWith('Heroku') diff --git a/src/types.ts b/src/types.ts index 067feeec..e457f674 100644 --- a/src/types.ts +++ b/src/types.ts @@ -242,6 +242,25 @@ export interface Element { group_id?: number } +export interface ElementGroup { + id: number + hash: string + team_id: number +} + +/** Usable Event model. */ +export interface Event { + id: number + event?: string + properties: Record + elements?: Element[] + timestamp: string + team_id: number + distinct_id: string + elements_hash: string + created_at: string +} + /** Properties shared by RawPerson and Person. */ export interface BasePerson { id: number @@ -276,6 +295,7 @@ export interface CohortPeople { cohort_id: number person_id: number } + export interface SessionRecordingEvent { uuid: string timestamp: string @@ -285,3 +305,12 @@ export interface SessionRecordingEvent { snapshot_data: string created_at: string } + +export interface PostgresSessionRecordingEvent extends Omit { + id: string +} + +export enum TimestampFormat { + ClickHouse = 'clickhouse', + ISO = 'iso', +} diff --git a/src/utils.ts b/src/utils.ts index e91ac52a..8a619069 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,7 +2,7 @@ import { Readable } from 'stream' import * as tar from 'tar-stream' import AdmZip from 'adm-zip' import * as zlib from 'zlib' -import { LogLevel } from './types' +import { LogLevel, TimestampFormat } from './types' import { randomBytes } from 'crypto' import { DateTime } from 'luxon' import { status } from './status' @@ -280,13 +280,23 @@ export class UUIDT extends UUID { } /** Format timestamp for ClickHouse. */ -export function castTimestampOrNow(timestamp?: DateTime | string | null): string { +export function castTimestampOrNow( + timestamp?: DateTime | string | null, + timestampFormat: TimestampFormat = TimestampFormat.ISO +): string { if (!timestamp) { timestamp = DateTime.utc() } else if (typeof timestamp === 'string') { timestamp = DateTime.fromISO(timestamp) } - return timestamp.toUTC().toFormat('yyyy-MM-dd HH:mm:ss.u') + timestamp = timestamp.toUTC() + if (timestampFormat === TimestampFormat.ClickHouse) { + return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') + } else if (timestampFormat === TimestampFormat.ISO) { + return timestamp.toUTC().toISO() + } else { + throw new Error(`Unrecognized timestamp format ${timestampFormat}!`) + } } export function delay(ms: number): Promise { diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 3ee2d62f..b86ed4be 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -1,54 +1,74 @@ -import { PluginsServer } from '../../src/types' -import { createServer } from '../../src/server' -import { resetTestDatabase } from '../helpers/sql' +import { + Element, + Event, + Person, + PersonDistinctId, + PluginsServer, + PluginsServerConfig, + PostgresSessionRecordingEvent, +} from '../../src/types' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { KafkaCollector, KafkaObserver } from '../helpers/kafka' import { UUIDT } from '../../src/utils' import { DateTime } from 'luxon' +import { createProcessEventTests } from '../shared/process-event' jest.setTimeout(180_000) // 3 minute timeout -let server: PluginsServer -let closeServer: () => Promise -const kafkaObserver = new KafkaObserver() +const extraServerConfig: Partial = { + KAFKA_ENABLED: true, + KAFKA_HOSTS: 'kafka:9092', + DATABASE_URL: 'postgres://posthog:posthog@localhost:5439/test_posthog', +} -beforeEach(async () => { - ;[server, closeServer] = await createServer() - await resetTestDatabase(`const processEvent = event => event`) - await resetTestDatabaseClickhouse() -}) -afterEach(() => { - closeServer() -}) +describe('process event (clickhouse)', () => { + const kafkaObserver = new KafkaObserver(extraServerConfig) -test('event is passed through', async () => { - const uuid = new UUIDT().toString() - const now = DateTime.utc() - console.log('starting kafka observer') - await kafkaObserver.start() - console.log('sending message') - const kafkaCollector = new KafkaCollector(kafkaObserver) - await kafkaObserver.handOffMessage({ - distinct_id: 'abcd', - ip: '1.1.1.1', - site_url: 'x.com', - team_id: 1, - uuid, - data: { + beforeEach(async () => { + await resetTestDatabaseClickhouse(extraServerConfig) + }) + + const server = createProcessEventTests( + 'clickhouse', + { + getSessionRecordingEvents: (server) => server.db.fetchSessionRecordingEvents(), + getEvents: (server) => server.db.fetchEvents(), + getPersons: (server) => server.db.fetchPersons(), + getDistinctIds: (server, person) => server.db.fetchDistinctIdValues(person), + getElements: (server) => server.db.fetchElements(), + }, + extraServerConfig + ) + + test('event is passed through', async () => { + const uuid = new UUIDT().toString() + const now = DateTime.utc() + console.log('starting kafka observer') + await kafkaObserver.start() + console.log('sending message') + const kafkaCollector = new KafkaCollector(kafkaObserver) + await kafkaObserver.handOffMessage({ distinct_id: 'abcd', ip: '1.1.1.1', site_url: 'x.com', team_id: 1, - now: now.toString(), - event: 'test', uuid, - }, - now, - sent_at: null, - }) - console.log('waiting for messages') - const processedMessages = await kafkaCollector.collect(1) + data: { + distinct_id: 'abcd', + ip: '1.1.1.1', + site_url: 'x.com', + team_id: 1, + now: now.toString(), + event: 'test', + uuid, + }, + now, + sent_at: null, + }) + console.log('waiting for messages') + const processedMessages = await kafkaCollector.collect(1) - console.log(processedMessages) - expect(1).toEqual(1) + console.log(processedMessages) + expect(1).toEqual(1) + }) }) diff --git a/tests/helpers/clickhouse.ts b/tests/helpers/clickhouse.ts index aef8f751..9a006dd2 100644 --- a/tests/helpers/clickhouse.ts +++ b/tests/helpers/clickhouse.ts @@ -1,12 +1,14 @@ import { defaultConfig } from '../../src/config' import { ClickHouse } from 'clickhouse' +import { PluginsServerConfig } from '../../src/types' -export async function resetTestDatabaseClickhouse(): Promise { +export async function resetTestDatabaseClickhouse(extraServerConfig: Partial): Promise { + const config = { ...defaultConfig, ...extraServerConfig } const clickhouse = new ClickHouse({ - url: `http://$${defaultConfig.CLICKHOUSE_HOST}`, + url: `http://$${config.CLICKHOUSE_HOST}`, port: 8123, config: { - database: defaultConfig.CLICKHOUSE_DATABASE, + database: config.CLICKHOUSE_DATABASE, }, }) await clickhouse.query('TRUNCATE events').toPromise() diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 37b7dbe7..15613922 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -2,8 +2,9 @@ import { EventEmitter } from 'events' import { Kafka, Consumer, logLevel, EachMessagePayload, Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' import { parseRawEventMessage } from '../../src/ingestion/utils' -import { EventMessage } from '../../src/types' +import { EventMessage, PluginsServerConfig } from '../../src/types' import { UUIDT } from '../../src/utils' +import { defaultConfig, overrideWithEnv } from '../../src/config' export class KafkaObserver extends EventEmitter { public kafka: Kafka @@ -12,11 +13,12 @@ export class KafkaObserver extends EventEmitter { private isStarted: boolean - constructor() { + constructor(extraServerConfig: Partial) { super() + const config = { ...overrideWithEnv(defaultConfig, process.env), ...extraServerConfig } this.kafka = new Kafka({ clientId: `plugin-server-test-${new UUIDT()}`, - brokers: process.env.KAFKA_HOSTS!.split(','), + brokers: (config.KAFKA_HOSTS || '').split(','), logLevel: logLevel.NOTHING, }) this.producer = this.kafka.producer() diff --git a/tests/helpers/plugins.ts b/tests/helpers/plugins.ts index 75f89e9b..224f3840 100644 --- a/tests/helpers/plugins.ts +++ b/tests/helpers/plugins.ts @@ -4,6 +4,8 @@ import path from 'path' import os from 'os' import AdmZip from 'adm-zip' +export const commonUserId = 1001 +export const commonOrganizationMembershipId = '0177364a-fc7b-0000-511c-137090b9e4e1' export const commonOrganizationId = 'ca30f2ec-e9a4-4001-bf27-3ef194086068' export const plugin60: Plugin = { diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 6e4463e0..92e39a95 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -1,49 +1,34 @@ -import { makePluginObjects, commonOrganizationId } from './plugins' +import { makePluginObjects, commonOrganizationId, commonUserId, commonOrganizationMembershipId } from './plugins' import { defaultConfig } from '../../src/config' import { Pool } from 'pg' import { delay, UUIDT } from '../../src/utils' +import { PluginsServerConfig } from '../../src/types' -export async function resetTestDatabase(code: string): Promise { - const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) +export async function resetTestDatabase( + code: string, + extraServerConfig: Partial = {} +): Promise { + const config = { ...defaultConfig, ...extraServerConfig } + const db = new Pool({ connectionString: config.DATABASE_URL }) const mocks = makePluginObjects(code) + await db.query('DELETE FROM posthog_element') + await db.query('DELETE FROM posthog_elementgroup') + await db.query('DELETE FROM posthog_sessionrecordingevent') + await db.query('DELETE FROM posthog_persondistinctid') + await db.query('DELETE FROM posthog_person') + await db.query('DELETE FROM posthog_event') await db.query('DELETE FROM posthog_pluginstorage') await db.query('DELETE FROM posthog_pluginattachment') await db.query('DELETE FROM posthog_pluginconfig') await db.query('DELETE FROM posthog_plugin') await db.query('DELETE FROM posthog_team') + await db.query('DELETE FROM posthog_organizationmembership') await db.query('DELETE FROM posthog_organization') + await db.query('DELETE FROM posthog_user') const teamIds = mocks.pluginConfigRows.map((c) => c.team_id) - await insertRow(db, 'posthog_organization', { - id: commonOrganizationId, - name: 'TEST ORG', - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - personalization: '{}', - }) - for (const teamId of teamIds) { - await insertRow(db, 'posthog_team', { - id: teamId, - organization_id: commonOrganizationId, - app_urls: [], - name: 'TEST PROJECT', - event_names: [], - event_names_with_usage: [], - event_properties: [], - event_properties_with_usage: [], - event_properties_numerical: [], - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - anonymize_ips: false, - completed_snippet_onboarding: true, - ingested_event: true, - uuid: new UUIDT().toString(), - session_recording_opt_in: true, - plugins_opt_in: true, - opt_out_capture: false, - is_demo: false, - }) - } + await createUserTeamAndOrganization(db, teamIds[0]) + for (const plugin of mocks.pluginRows) { await insertRow(db, 'posthog_plugin', plugin) } @@ -71,3 +56,59 @@ async function insertRow(db: Pool, table: string, object: Record): throw error } } + +export async function createUserTeamAndOrganization( + db: Pool, + teamId: number, + userId: number = commonUserId, + organizationId: string = commonOrganizationId, + organizationMembershipId: string = commonOrganizationMembershipId +): Promise { + await insertRow(db, 'posthog_user', { + id: userId, + password: 'gibberish', + first_name: 'PluginTest', + last_name: 'User', + email: `test${userId}@posthog.com`, + distinct_id: `plugin_test_user_distinct_id_${userId}`, + is_staff: false, + is_active: false, + date_joined: new Date().toISOString(), + }) + await insertRow(db, 'posthog_organization', { + id: organizationId, + name: 'TEST ORG', + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + personalization: '{}', + }) + await insertRow(db, 'posthog_organizationmembership', { + id: organizationMembershipId, + organization_id: organizationId, + user_id: userId, + level: 15, + joined_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }) + await insertRow(db, 'posthog_team', { + id: teamId, + organization_id: organizationId, + app_urls: [], + name: 'TEST PROJECT', + event_names: JSON.stringify([]), + event_names_with_usage: JSON.stringify([]), + event_properties: JSON.stringify([]), + event_properties_with_usage: JSON.stringify([]), + event_properties_numerical: JSON.stringify([]), + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + anonymize_ips: false, + completed_snippet_onboarding: true, + ingested_event: true, + uuid: new UUIDT().toString(), + session_recording_opt_in: true, + plugins_opt_in: true, + opt_out_capture: false, + is_demo: false, + }) +} diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts new file mode 100644 index 00000000..d404d826 --- /dev/null +++ b/tests/postgres/process-event.test.ts @@ -0,0 +1,14 @@ +import { PluginsServer, Event, Person, PersonDistinctId, Element, PostgresSessionRecordingEvent } from '../../src/types' +import { createProcessEventTests } from '../shared/process-event' + +jest.setTimeout(600000) // 600 sec timeout + +describe('process event (postgresql)', () => { + createProcessEventTests('postgresql', { + getSessionRecordingEvents: (server) => server.db.fetchSessionRecordingEvents(), + getEvents: (server) => server.db.fetchEvents(), + getPersons: (server) => server.db.fetchPersons(), + getDistinctIds: (server, person) => server.db.fetchDistinctIdValues(person), + getElements: (server) => server.db.fetchElements(), + }) +}) diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index ef459fc1..dfa6ff41 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -714,9 +714,9 @@ test('posthog in runEvery', async () => { expect(Client).toHaveBeenCalledTimes(2) expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue - expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue - const mockClientInstance = (Client as any).mock.instances[0] + const mockClientInstance = (Client as any).mock.instances[1] const mockSendTask = mockClientInstance.sendTask expect(mockSendTask.mock.calls[0][0]).toEqual('posthog.tasks.process_event.process_event_with_plugins') @@ -753,9 +753,9 @@ test('posthog in runEvery with timestamp', async () => { expect(Client).toHaveBeenCalledTimes(2) expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue - expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue - const mockClientInstance = (Client as any).mock.instances[0] + const mockClientInstance = (Client as any).mock.instances[1] const mockSendTask = mockClientInstance.sendTask expect(mockSendTask.mock.calls[0][0]).toEqual('posthog.tasks.process_event.process_event_with_plugins') diff --git a/tests/shared/process-event.ts b/tests/shared/process-event.ts new file mode 100644 index 00000000..1417dc52 --- /dev/null +++ b/tests/shared/process-event.ts @@ -0,0 +1,1045 @@ +import { PluginEvent } from '@posthog/plugin-scaffold/src/types' +import { createServer } from '../../src/server' +import { + LogLevel, + PluginsServer, + Team, + Event, + Person, + Element, + PostgresSessionRecordingEvent, + PluginsServerConfig, +} from '../../src/types' +import { createUserTeamAndOrganization, resetTestDatabase } from '../helpers/sql' +import { EventsProcessor } from '../../src/ingestion/process-event' +import { DateTime } from 'luxon' +import { UUIDT } from '../../src/utils' + +jest.setTimeout(600000) // 600 sec timeout + +async function getTeams(server: PluginsServer): Promise { + return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows +} + +async function getFirstTeam(server: PluginsServer): Promise { + return (await getTeams(server))[0] +} + +async function createPerson( + server: PluginsServer, + team: Team, + distinctIds: string[], + properties: Record = {} +): Promise { + return server.db.createPerson(DateTime.utc(), properties, team.id, null, false, new UUIDT().toString(), distinctIds) +} + +export const createProcessEventTests = ( + database: 'postgresql' | 'clickhouse', + { + getSessionRecordingEvents, + getEvents, + getPersons, + getDistinctIds, + getElements, + }: { + getSessionRecordingEvents: (server: PluginsServer) => Promise + getEvents: (server: PluginsServer) => Promise + getPersons: (server: PluginsServer) => Promise + getDistinctIds: (server: PluginsServer, person: Person) => Promise + getElements: (server: PluginsServer, event: Event) => Promise + }, + extraServerConfig?: Partial +): PluginsServer => { + let queryCounter = 0 + let team: Team + let server: PluginsServer + let stopServer: () => Promise + let eventsProcessor: EventsProcessor + let now = DateTime.utc() + + async function getServer(): Promise<[PluginsServer, () => Promise]> { + const [server, stopServer] = await createServer({ + PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', + CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', + LOG_LEVEL: LogLevel.Log, + ...(extraServerConfig ?? {}), + }) + + await server.redis.del(server.PLUGINS_CELERY_QUEUE) + await server.redis.del(server.CELERY_DEFAULT_QUEUE) + + const query = server.postgres.query.bind(server.postgres) + server.postgres.query = (queryText: any, values?: any, callback?: any): any => { + queryCounter++ + return query(queryText, values, callback) + } + + return [server, stopServer] + } + + beforeEach(async () => { + const testCode = ` + function processEvent (event, meta) { + event.properties["somewhere"] = "over the rainbow"; + return event + } + ` + await resetTestDatabase(testCode, extraServerConfig) + ;[server, stopServer] = await getServer() + eventsProcessor = new EventsProcessor(server) + queryCounter = 0 + team = await getFirstTeam(server) + now = DateTime.utc() + }) + + afterEach(async () => { + await stopServer?.() + }) + + test('capture new person', async () => { + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [true, team.id]) + team = await getFirstTeam(server) + + expect(team.event_names).toEqual([]) + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '๐Ÿ’ป' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(queryCounter).toBe(12) + + // TODO: 12 vs 28 is a big difference. Why so? + // num_queries = 28 + // if settings.EE_AVAILABLE: # extra queries to check for hooks + // num_queries += 4 + // if settings.MULTI_TENANCY: # extra query to check for billing plan + // num_queries += 1 + // with self.assertNumQueries(num_queries): + + // capture a second time to verify e.g. event_names is not ['$autocapture', '$autocapture'] + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '๐Ÿ’ป' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const events = await getEvents(server) + const persons = await getPersons(server) + expect(events.length).toEqual(2) + expect(persons.length).toEqual(1) + + const [event] = events + const [person] = persons + const distinctIds = await getDistinctIds(server, person) + + expect(event.distinct_id).toEqual('2') + expect(distinctIds).toEqual(['2']) + expect(event.event).toEqual('$autocapture') + expect(event.elements_hash).toEqual('0679137c0cd2408a2906839143e7a71f') + + const elements = await getElements(server, event) + expect(elements[0].tag_name).toEqual('a') + expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) + expect(elements[1].order).toEqual(1) + expect(elements[1].text).toEqual('๐Ÿ’ป') + + team = await getFirstTeam(server) + expect(team.event_names).toEqual(['$autocapture']) + expect(team.event_names_with_usage).toEqual([{ event: '$autocapture', volume: null, usage_count: null }]) + expect(team.event_properties).toEqual(['distinct_id', 'token', '$ip']) + expect(team.event_properties_with_usage).toEqual([ + { key: 'distinct_id', usage_count: null, volume: null }, + { key: 'token', usage_count: null, volume: null }, + { key: '$ip', usage_count: null, volume: null }, + ]) + }) + + test('capture no element', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual(['asdfasdfasdf']) + const [event] = await getEvents(server) + expect(event.event).toBe('$pageview') + }) + + test('capture sent_at', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture sent_at no timezones', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }).setZone('UTC+4') + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }).setZone('UTC+4') + + // TODO: not sure if this is correct? + // tomorrow = tomorrow.replace(tzinfo=None) + // tomorrow_sent_at = tomorrow_sent_at.replace(tzinfo=None) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow, + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture no sent_at', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + null, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + const difference = tomorrow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + expect(difference).toBeLessThan(1) + }) + + test('ip capture', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.properties['$ip']).toBe('11.12.13.14') + }) + + test('ip override', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { $ip: '1.0.0.1', distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.properties['$ip']).toBe('1.0.0.1') + }) + + test('anonymized ip capture', async () => { + await server.db.postgresQuery('update posthog_team set anonymize_ips = $1', [true]) + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.properties['$ip']).not.toBeDefined() + }) + + test('alias', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('alias reverse', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'old_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'old_distinct_id', token: team.api_token, alias: 'new_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('alias twice', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getPersons(server)).length).toBe(1) + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + + await createPerson(server, team, ['old_distinct_id_2']) + expect((await getPersons(server)).length).toBe(2) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id_2' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(2) + expect((await getPersons(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + 'old_distinct_id_2', + ]) + }) + + test('alias before person', async () => { + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect((await getPersons(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'new_distinct_id', + 'old_distinct_id', + ]) + }) + + test('alias both existing', async () => { + await createPerson(server, team, ['old_distinct_id']) + await createPerson(server, team, ['new_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('offset timestamp', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') + }) + + test('offset timestamp no sent_at', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + null, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') + }) + + test('alias merge properties', async () => { + await createPerson(server, team, ['old_distinct_id'], { + key_on_both: 'old value both', + key_on_old: 'old value', + }) + await createPerson(server, team, ['new_distinct_id'], { + key_on_both: 'new value both', + key_on_new: 'new value', + }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect((await getPersons(server)).length).toBe(1) + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['old_distinct_id', 'new_distinct_id']) + expect(person.properties).toEqual({ + key_on_both: 'new value both', + key_on_new: 'new value', + key_on_old: 'old value', + }) + }) + + test('long htext', async () => { + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 'new_distinct_id', + token: team.api_token, + $elements: [ + { + tag_name: 'a', + $el_text: 'a'.repeat(2050), + attr__href: 'a'.repeat(2050), + nth_child: 1, + nth_of_type: 2, + attr__class: 'btn btn-sm', + }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + expect(event.elements_hash).toEqual('c2659b28e72835706835764cf7f63c2a') + const [element] = await getElements(server, event) + expect(element.href?.length).toEqual(2048) + expect(element.text?.length).toEqual(400) + }) + + test('capture first team event', async () => { + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [false, team.id]) + + eventsProcessor.posthog = { + identify: jest.fn((distinctId) => true), + capture: jest.fn((event, properties) => true), + } as any + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 1, + token: team.api_token, + $elements: [{ tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(eventsProcessor.posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id_1001') + expect(eventsProcessor.posthog.capture).toHaveBeenCalledWith('first team event ingested', { + team: team.uuid, + }) + + team = await getFirstTeam(server) + expect(team.ingested_event).toEqual(true) + + const [event] = await getEvents(server) + expect(event.elements_hash).toEqual('a89021a60b3497d24e93ae181fba01aa') + }) + + test('snapshot event stored as session_recording_event', async () => { + await eventsProcessor.processEvent( + 'some-id', + '', + '', + ({ + event: '$snapshot', + properties: { $session_id: 'abcf-efg', $snapshot_data: { timestamp: 123 } }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const events = await getEvents(server) + expect(events.length).toEqual(0) + + const sessionRecordingEvents = await getSessionRecordingEvents(server) + expect(sessionRecordingEvents.length).toBe(1) + + const [event] = sessionRecordingEvents + expect(event.session_id).toEqual('abcf-efg') + expect(event.distinct_id).toEqual('some-id') + expect(event.snapshot_data).toEqual({ timestamp: 123 }) + }) + + test('identify set', async () => { + await createPerson(server, team, ['distinct_id']) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.properties['$set']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(2) + const [person2] = await getPersons(server) + expect(person2.properties).toEqual({ a_prop: 'test-2', b_prop: 'test-2b', c_prop: 'test-1' }) + }) + + test('identify set_once', async () => { + await createPerson(server, team, ['distinct_id']) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.properties['$set_once']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(2) + const [person2] = await getPersons(server) + expect(person2.properties).toEqual({ a_prop: 'test-1', b_prop: 'test-2b', c_prop: 'test-1' }) + }) + + test('distinct with anonymous_id', async () => { + await createPerson(server, team, ['anonymous_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + const [event] = await getEvents(server) + expect(event.properties['$set']).toEqual({ a_prop: 'test' }) + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test' }) + + // check no errors as this call can happen multiple times + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + }) + + // This case is likely to happen after signup, for example: + // 1. User browses website with anonymous_id + // 2. User signs up, triggers event with their new_distinct_id (creating a new Person) + // 3. In the frontend, try to alias anonymous_id with new_distinct_id + // Result should be that we end up with one Person with both ID's + test('distinct with anonymous_id which was already created', async () => { + await createPerson(server, team, ['anonymous_id']) + await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties['email']).toEqual('someone@gmail.com') + }) + + test('distinct with multiple anonymous_ids which were already created', async () => { + await createPerson(server, team, ['anonymous_id']) + await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const persons1 = await getPersons(server) + expect(persons1.length).toBe(1) + expect(await getDistinctIds(server, persons1[0])).toEqual(['anonymous_id', 'new_distinct_id']) + expect(persons1[0].properties['email']).toEqual('someone@gmail.com') + + await createPerson(server, team, ['anonymous_id_2']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id_2', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const persons2 = await getPersons(server) + expect(persons2.length).toBe(1) + expect(await getDistinctIds(server, persons2[0])).toEqual(['anonymous_id', 'new_distinct_id', 'anonymous_id_2']) + expect(persons2[0].properties['email']).toEqual('someone@gmail.com') + }) + + test('distinct team leakage', async () => { + await createUserTeamAndOrganization( + server.postgres, + 3, + 1002, + '01774e2f-0d01-0000-ee94-9a238640c6ee', + '0174f81e-36f5-0000-7ef8-cc26c1fbab1c' + ) + const team2 = (await getTeams(server))[1] + await createPerson(server, team2, ['2'], { email: 'team2@gmail.com' }) + await createPerson(server, team, ['1', '2']) + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: '1', + token: team.api_token, + distinct_id: '2', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const people = await getPersons(server) + expect(people.length).toEqual(2) + expect(people[1].team_id).toEqual(team.id) + expect(people[1].properties).toEqual({}) + expect(await getDistinctIds(server, people[1])).toEqual(['1', '2']) + expect(people[0].team_id).toEqual(team2.id) + expect(await getDistinctIds(server, people[0])).toEqual(['2']) + }) + + test('set is_identified', async () => { + const distinct_id = '777' + const person1 = await createPerson(server, team, [distinct_id]) + expect(person1.is_identified).toBe(false) + + await eventsProcessor.processEvent( + distinct_id, + '', + '', + ({ event: '$identify', properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person2] = await getPersons(server) + expect(person2.is_identified).toBe(true) + }) + + test('team event_properties', async () => { + expect(team.event_properties_numerical).toEqual([]) + + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: 'purchase', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + team = await getFirstTeam(server) + expect(team.event_properties).toEqual(['price', 'name', '$ip']) + expect(team.event_properties_numerical).toEqual(['price']) + }) + + test('event name object json', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: { 'event name': 'as object' }, properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.event).toEqual('{"event name":"as object"}') + }) + + test('event name array json', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: ['event name', 'a list'], properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.event).toEqual('["event name","a list"]') + }) + + test('long event name substr', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + 'uuid' + ) + + const [event] = await getEvents(server) + expect(event.event?.length).toBe(200) + }) + + return server! +} From 5a1e69e1e48a321d372646fd9381b3af7e4171b5 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 11:58:54 +0100 Subject: [PATCH 206/218] postgres e2e ingestion test (#120) --- src/config.ts | 2 + src/types.ts | 1 + src/worker/queue.ts | 33 ++++++++++----- tests/clickhouse/process-event.test.ts | 10 +---- tests/postgres/e2e.test.ts | 56 ++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 tests/postgres/e2e.test.ts diff --git a/src/config.ts b/src/config.ts index 1b6c108f..d3e08dee 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,6 +20,7 @@ export function getDefaultConfig(): PluginsServerConfig { KAFKA_CLIENT_CERT_B64: null, KAFKA_CLIENT_CERT_KEY_B64: null, KAFKA_TRUSTED_CERT_B64: null, + PLUGIN_SERVER_INGESTION: false, PLUGINS_CELERY_QUEUE: 'posthog-plugins', REDIS_URL: 'redis://127.0.0.1', BASE_DIR: '.', @@ -40,6 +41,7 @@ export function getDefaultConfig(): PluginsServerConfig { export function getConfigHelp(): Record { return { + PLUGIN_SERVER_INGESTION: 'Ingest events via plugin-server', CELERY_DEFAULT_QUEUE: 'Celery outgoing queue', PLUGINS_CELERY_QUEUE: 'Celery incoming queue', DATABASE_URL: 'Postgres database URL', diff --git a/src/types.ts b/src/types.ts index e457f674..5aeb9416 100644 --- a/src/types.ts +++ b/src/types.ts @@ -42,6 +42,7 @@ export interface PluginsServerConfig extends Record { WEB_PORT: number WEB_HOSTNAME: string LOG_LEVEL: LogLevel + PLUGIN_SERVER_INGESTION: boolean SENTRY_DSN: string | null STATSD_HOST: string | null STATSD_PORT: number diff --git a/src/worker/queue.ts b/src/worker/queue.ts index 99ae5ed2..f55f1995 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -6,6 +6,7 @@ import Client from '../celery/client' import { PluginsServer, Queue } from '../types' import { KafkaQueue } from '../ingestion/kafka-queue' import { status } from '../status' +import { UUIDT } from '../utils' export async function startQueue( server: PluginsServer, @@ -45,15 +46,29 @@ async function startQueueRedis( const processedEvent = await processEvent(event) if (processedEvent) { const { distinct_id, ip, site_url, team_id, now, sent_at, ...data } = processedEvent - client.sendTask('posthog.tasks.process_event.process_event', [], { - distinct_id, - ip, - site_url, - data, - team_id, - now, - sent_at, - }) + + if (server.PLUGIN_SERVER_INGESTION) { + await server.eventsProcessor.processEvent( + distinct_id, + ip, + site_url, + processedEvent, + team_id, + DateTime.fromISO(now), + sent_at ? DateTime.fromISO(sent_at) : null, + new UUIDT().toString() + ) + } else { + client.sendTask('posthog.tasks.process_event.process_event', [], { + distinct_id, + ip, + site_url, + data, + team_id, + now, + sent_at, + }) + } } } catch (e) { Sentry.captureException(e) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index b86ed4be..96214920 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -1,12 +1,4 @@ -import { - Element, - Event, - Person, - PersonDistinctId, - PluginsServer, - PluginsServerConfig, - PostgresSessionRecordingEvent, -} from '../../src/types' +import { PluginsServerConfig } from '../../src/types' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { KafkaCollector, KafkaObserver } from '../helpers/kafka' import { UUIDT } from '../../src/utils' diff --git a/tests/postgres/e2e.test.ts b/tests/postgres/e2e.test.ts new file mode 100644 index 00000000..208eef5a --- /dev/null +++ b/tests/postgres/e2e.test.ts @@ -0,0 +1,56 @@ +import { LogLevel } from '../../src/types' +import { resetTestDatabase } from '../helpers/sql' +import { startPluginsServer } from '../../src/server' +import { makePiscina } from '../../src/worker/piscina' +import { PluginsServer } from '../../src/types' +import { createPosthog, DummyPostHog } from '../../src/extensions/posthog' +import { pluginConfig39 } from '../helpers/plugins' +import { delay } from '../../src/utils' + +jest.setTimeout(60000) // 60 sec timeout + +describe('e2e postgres ingestion', () => { + let server: PluginsServer + let stopServer: () => Promise + let posthog: DummyPostHog + + beforeEach(async () => { + await resetTestDatabase(` + async function processEvent (event) { + event.properties.processed = 'hell yes' + return event + } + `) + const startResponse = await startPluginsServer( + { + WORKER_CONCURRENCY: 2, + PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', + CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', + PLUGIN_SERVER_INGESTION: true, + LOG_LEVEL: LogLevel.Log, + KAFKA_ENABLED: false, + }, + makePiscina + ) + server = startResponse.server + stopServer = startResponse.stop + + await server.redis.del(server.PLUGINS_CELERY_QUEUE) + await server.redis.del(server.CELERY_DEFAULT_QUEUE) + + posthog = createPosthog(server, pluginConfig39) + }) + + afterEach(async () => { + await stopServer() + }) + + test('event captured, processed, ingested', async () => { + expect((await server.db.fetchEvents()).length).toBe(0) + posthog.capture('custom event', { name: 'haha' }) + await delay(2000) + const events = await server.db.fetchEvents() + expect(events.length).toBe(1) + expect(events[0].properties.processed).toEqual('hell yes') + }) +}) From 1efbc9ba56082aa863c42b04452ede9d61022170 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 3 Feb 2021 01:09:03 +0100 Subject: [PATCH 207/218] Kafka e2e tests (#121) * kafka e2e test * kafka e2e test * kafka host from env * get some ch process event tests working * more working clickhouse tests * wait a bit longer * add chainToElements, fix elementsToString bug * remove quotes from inside sanitizeSqlIdentifier to also work with clickhouse * split dev setup command * fetch elements from clickhouse * more elements * bugfix * refactor and reuse wait function * ingest kafka events with the right structure * remove leftover * fix clickhouse timestamp * simplify process event creation and call the methods directly * fix uuid test * refactor delayed event fetching to support session recording events * catch bad uuids * wait for session recording events in test * use right timestamp for session recording events * use the same database as posthog (the app) * use local db * deserialize clickhouse session recording events * split dev scripts * try to make tests work by specifying db * increase kafka log level * cleanup * pass idl protos to clickhouse in github actions * start the clickhouse container in another step * let's try like this * WIP * WIP * sudo * also alias zookeeper * export zookeeper * use docker-compose.ch.yml * detached * element group test * create tests * debug test * remove some redundancy * reduce some noise * try to make topics * compatible with posthog migration 0122 * hide error * try to close e2e open handles * reuse kafkaProducer on server * Add DB.clickhouseQuery * Put isUUIDFormat on the UUID class Co-authored-by: Michael Matloka --- .github/workflows/ci.yml | 29 +- package.json | 8 +- src/celery/worker.ts | 4 +- src/config.ts | 2 +- src/db.ts | 122 ++++++-- src/extensions/posthog.ts | 7 +- src/ingestion/kafka-queue.ts | 15 +- src/ingestion/process-event.ts | 54 +--- src/ingestion/utils.ts | 60 +++- src/server.ts | 8 +- src/types.ts | 9 +- src/utils.ts | 22 +- tests/clickhouse/e2e.test.ts | 61 ++++ tests/clickhouse/ingestion-utils.test.ts | 50 ++++ tests/clickhouse/process-event.test.ts | 57 +--- tests/helpers/kafka.ts | 83 +++++- tests/helpers/sql.ts | 1 + tests/postgres/e2e.test.ts | 10 +- tests/postgres/process-event.test.ts | 54 +++- tests/shared/process-event.ts | 353 ++++++++++++++--------- tests/utils.test.ts | 2 +- 21 files changed, 708 insertions(+), 303 deletions(-) create mode 100644 tests/clickhouse/e2e.test.ts create mode 100644 tests/clickhouse/ingestion-utils.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faeda4cf..d0030629 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,29 +122,13 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - clickhouse: - image: yandex/clickhouse-server - ports: - - '8123:8123' - - '9000:9000' - - '9440:9440' - - '9009:9009' - zookeeper: - image: wurstmeister/zookeeper - kafka: - image: wurstmeister/kafka - ports: - - '9092:9092' - env: - KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 env: REDIS_URL: 'redis://localhost' CLICKHOUSE_HOST: 'localhost' CLICKHOUSE_DATABASE: 'posthog_test' KAFKA_ENABLED: 'true' - KAFKA_HOSTS: 'localhost:9092' + KAFKA_HOSTS: 'kafka:9092' steps: - name: Check out Django server for database setup @@ -158,6 +142,17 @@ jobs: with: path: 'plugin-server' + - name: Fix Kafka Hostname + run: | + sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts' + ping -c 1 kafka + ping -c 1 zookeeper + + - name: Start Kafka, Clickhouse, Zookeeper + run: | + cd posthog/ee + docker-compose -f docker-compose.ch.yml up -d zookeeper kafka clickhouse + - name: Set up Python uses: actions/setup-python@v2 with: diff --git a/package.json b/package.json index fbd50b6e..d314c9cf 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", "start:dev": "ts-node-dev --exit-child src/index.ts --base-dir ../posthog", - "start:dev:ee": "DATABASE_URL=postgres://posthog:posthog@localhost:5439/posthog KAFKA_ENABLED=true KAFKA_HOSTS=localhost:9092 yarn start:dev", + "start:dev:ee": "KAFKA_ENABLED=true KAFKA_HOSTS=localhost:9092 yarn start:dev", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && eslint --fix . && prettier --write .", @@ -24,8 +24,10 @@ "prettier:check": "prettier --check .", "prepare": "yarn compile:protobuf", "prepublishOnly": "yarn build", - "setup:dev": "cd ../posthog && (dropdb test_posthog || echo 'no db to drop') && createdb test_posthog && source env/bin/activate && DATABASE_URL=postgres://localhost:5432/test_posthog DEBUG=1 python manage.py migrate", - "setup:dev:ee": "export DEBUG=1 PRIMARY_DB=clickhouse DATABASE_URL=postgres://posthog:posthog@localhost:5439/test_posthog PGPASSWORD=posthog && cd ../posthog && (dropdb -p 5439 -h localhost -U posthog test_posthog || echo 'no db to drop') && createdb -p 5439 -h localhost -U posthog test_posthog && source env/bin/activate && python manage.py migrate && python manage.py migrate_clickhouse" + "setup:dev:clickhouse": "cd ../posthog && export DEBUG=1 PRIMARY_DB=clickhouse && source env/bin/activate && python manage.py migrate_clickhouse", + "setup:test:ee": "yarn setup:test:postgres && yarn setup:test:clickhouse", + "setup:test:postgres": "cd ../posthog && (dropdb test_posthog || echo 'no db to drop') && createdb test_posthog && source env/bin/activate && DATABASE_URL=postgres://localhost:5432/test_posthog DEBUG=1 python manage.py migrate", + "setup:test:clickhouse": "cd ../posthog && export TEST=1 PRIMARY_DB=clickhouse CLICKHOUSE_DATABASE=posthog_test && source env/bin/activate && python manage.py migrate_clickhouse" }, "bin": { "posthog-plugin-server": "bin/posthog-plugin-server" diff --git a/src/celery/worker.ts b/src/celery/worker.ts index 19d96f7f..107dad04 100644 --- a/src/celery/worker.ts +++ b/src/celery/worker.ts @@ -171,14 +171,14 @@ export class Worker extends Base implements Queue { throw new Error(`Missing process handler for task ${taskName}`) } - console.info( + console.debug( `celery.node Received task: ${taskName}[${taskId}], args: ${args}, kwargs: ${JSON.stringify(kwargs)}` ) const timeStart = process.hrtime() const taskPromise = handler(...args, kwargs).then((result) => { const diff = process.hrtime(timeStart) - console.info( + console.debug( `celery.node Task ${taskName}[${taskId}] succeeded in ${diff[0] + diff[1] / 1e9}s: ${result}` ) this.activeTasks.delete(taskPromise) diff --git a/src/config.ts b/src/config.ts index d3e08dee..a1758dc4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,7 +10,7 @@ export function getDefaultConfig(): PluginsServerConfig { CELERY_DEFAULT_QUEUE: 'celery', DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/test_posthog' : 'postgres://localhost:5432/posthog', CLICKHOUSE_HOST: 'localhost', - CLICKHOUSE_DATABASE: 'default', + CLICKHOUSE_DATABASE: isTestEnv ? 'posthog_test' : 'default', CLICKHOUSE_USERNAME: 'default', CLICKHOUSE_PASSWORD: null, CLICKHOUSE_CA: null, diff --git a/src/db.ts b/src/db.ts index 7b4f430c..98f635dc 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,20 +1,24 @@ import { Properties } from '@posthog/plugin-scaffold' -import { ClickHouse } from 'clickhouse' +import { ClickHouse, QueryCursor } from 'clickhouse' import { Producer } from 'kafkajs' import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' +import { string } from 'yargs' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' -import { unparsePersonPartial } from './ingestion/utils' +import { chainToElements, hashElements, unparsePersonPartial } from './ingestion/utils' import { Person, PersonDistinctId, RawPerson, RawOrganization, - Team, PostgresSessionRecordingEvent, Event, + ClickHouseEvent, + Element, + SessionRecordingEvent, + ElementGroup, } from './types' -import { castTimestampOrNow, sanitizeSqlIdentifier, UUIDT } from './utils' +import { castTimestampOrNow, clickHouseTimestampToISO, sanitizeSqlIdentifier } from './utils' /** The recommended way of accessing the database. */ export class DB { @@ -31,6 +35,8 @@ export class DB { this.clickhouse = clickhouse } + // Direct queries + public async postgresQuery( queryTextOrConfig: string | QueryConfig, values?: I @@ -38,6 +44,13 @@ export class DB { return this.postgres.query(queryTextOrConfig, values) } + public async clickhouseQuery(query: string, reqParams?: Record): Promise> { + if (!this.clickhouse) { + throw new Error('ClickHouse connection has not been provided to this DB instance!') + } + return this.clickhouse.query(query, reqParams).toPromise() + } + // Person public async fetchPersons(): Promise { @@ -106,7 +119,7 @@ export class DB { const values = [...Object.values(unparsePersonPartial(update)), person.id] await this.postgresQuery( `UPDATE posthog_person SET ${Object.keys(update).map( - (field, index) => sanitizeSqlIdentifier(field) + ' = $' + (index + 1) + (field, index) => `"${sanitizeSqlIdentifier(field)}" = $${index + 1}` )} WHERE id = $${Object.values(update).length + 1}`, values ) @@ -130,10 +143,8 @@ export class DB { await this.postgresQuery('DELETE FROM posthog_persondistinctid WHERE person_id = $1', [personId]) await this.postgresQuery('DELETE FROM posthog_person WHERE id = $1', [personId]) if (this.clickhouse) { - await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() - await this.clickhouse - .query(`ALTER TABLE person_distinct_id DELETE WHERE person_id = ${personId}`) - .toPromise() + await this.clickhouseQuery(`ALTER TABLE person DELETE WHERE id = ${personId}`) + await this.clickhouseQuery(`ALTER TABLE person_distinct_id DELETE WHERE person_id = ${personId}`) } } @@ -168,7 +179,7 @@ export class DB { const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } await this.postgresQuery( `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( - (field, index) => sanitizeSqlIdentifier(field) + ' = $' + (index + 1) + (field, index) => `"${sanitizeSqlIdentifier(field)}" = $${index + 1}` )} WHERE id = $${Object.values(update).length + 1}`, [...Object.values(update), personDistinctId.id] ) @@ -192,21 +203,96 @@ export class DB { // Event - public async fetchEvents(): Promise { - const result = await this.postgresQuery('SELECT * FROM posthog_event') - return result.rows as Event[] + public async fetchEvents(): Promise { + if (this.kafkaProducer) { + const events = (await this.clickhouseQuery(`SELECT * FROM events`)) as ClickHouseEvent[] + return ( + events?.map( + (event) => + ({ + ...event, + ...(typeof event['properties'] === 'string' + ? { properties: JSON.parse(event.properties) } + : {}), + timestamp: clickHouseTimestampToISO(event.timestamp), + } as ClickHouseEvent) + ) || [] + ) + } else { + const result = await this.postgresQuery('SELECT * FROM posthog_event') + return result.rows as Event[] + } } // SessionRecordingEvent - public async fetchSessionRecordingEvents(): Promise { - const result = await this.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') - return result.rows as PostgresSessionRecordingEvent[] + public async fetchSessionRecordingEvents(): Promise { + if (this.kafkaProducer) { + const events = ((await this.clickhouseQuery( + `SELECT * FROM session_recording_events` + )) as SessionRecordingEvent[]).map((event) => { + return { + ...event, + snapshot_data: event.snapshot_data ? JSON.parse(event.snapshot_data) : null, + } + }) + return events + } else { + const result = await this.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') + return result.rows as PostgresSessionRecordingEvent[] + } } // Element - public async fetchElements(): Promise { - return (await this.postgresQuery('SELECT * FROM posthog_element')).rows + public async fetchElements(event?: Event): Promise { + if (this.kafkaProducer) { + const events = (await this.clickhouseQuery( + `SELECT elements_chain FROM events WHERE uuid='${sanitizeSqlIdentifier((event as any).uuid)}'` + )) as ClickHouseEvent[] + const chain = events?.[0]?.elements_chain + return chainToElements(chain) + } else { + return (await this.postgresQuery('SELECT * FROM posthog_element')).rows + } + } + + public async createElementGroup(elements: Element[], teamId: number): Promise { + const cleanedElements = elements.map((element, index) => ({ ...element, order: index })) + const hash = hashElements(cleanedElements) + + try { + const insertResult = await this.postgresQuery( + 'INSERT INTO posthog_elementgroup (hash, team_id) VALUES ($1, $2) RETURNING *', + [hash, teamId] + ) + const elementGroup = insertResult.rows[0] as ElementGroup + for (const element of cleanedElements) { + await this.postgresQuery( + 'INSERT INTO posthog_element (text, tag_name, href, attr_id, nth_child, nth_of_type, attributes, "order", event_id, attr_class, group_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)', + [ + element.text, + element.tag_name, + element.href, + element.attr_id, + element.nth_child, + element.nth_of_type, + element.attributes || '{}', + element.order, + element.event_id, + element.attr_class, + elementGroup.id, + ] + ) + } + } catch (error) { + // Throw further if not postgres error nr "23505" == "unique_violation" + // https://www.postgresql.org/docs/12/errcodes-appendix.html + if (error.code !== '23505') { + throw error + } + } + + return hash } } diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index 51a09b04..94af08ca 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -26,10 +26,11 @@ export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig) if (server.KAFKA_ENABLED) { // Sending event to our Kafka>ClickHouse pipeline - const producer = server.kafka!.producer() sendEvent = async (data) => { - await producer.connect() - producer!.send({ + if (!server.kafkaProducer) { + throw new Error('kafkaProducer not configured!') + } + server.kafkaProducer.send({ topic: KAFKA_EVENTS_INGESTION_HANDOFF, messages: [ { diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index e5d68be3..26d1cc69 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -47,11 +47,16 @@ export class KafkaQueue implements Queue { ...rawEvent, data: JSON.parse(rawEvent.data), })) - const pluginEvents: PluginEvent[] = parsedEvents.map((parsedEvent) => ({ - ...parsedEvent, - event: parsedEvent.data.event, - properties: parsedEvent.data.properties, - })) + const pluginEvents: PluginEvent[] = rawEvents.map((rawEvent) => { + const { data: dataStr, ...restOfRawEvent } = rawEvent + const event = { ...restOfRawEvent, ...JSON.parse(dataStr) } + return { + ...event, + kafka_offset: restOfRawEvent.kafka_offset, + site_url: event.site_url || null, + ip: event.ip || null, + } + }) const processedEvents: PluginEvent[] = ( await this.processEventBatch(pluginEvents) ).filter((event: PluginEvent[] | false | null | undefined) => Boolean(event)) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 73a8e0db..304fd9d2 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -3,7 +3,6 @@ import { DateTime, Duration } from 'luxon' import { CohortPeople, Element, - ElementGroup, Person, PersonDistinctId, PluginsServer, @@ -12,11 +11,11 @@ import { Team, TimestampFormat, } from '../types' -import { castTimestampOrNow, UUIDT } from '../utils' +import { castTimestampOrNow, UUID, UUIDT } from '../utils' import { Event as EventProto, IEvent } from '../idl/protos' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' -import { elementsToString, hashElements, sanitizeEventName } from './utils' +import { elementsToString, sanitizeEventName } from './utils' import { ClickHouse } from 'clickhouse' import { DB } from '../db' import { status } from '../status' @@ -54,6 +53,9 @@ export class EventsProcessor { sentAt: DateTime | null, eventUuid: string ): Promise { + if (!UUID.validateString(eventUuid, false)) { + throw new Error(`Not a valid UUID: "${eventUuid}"`) + } const singleSaveTimer = new Date() const properties: Properties = data.properties ?? {} @@ -462,7 +464,7 @@ export class EventsProcessor { } else { let elementsHash = '' if (elements && elements.length > 0) { - elementsHash = await this.createElementGroup(elements, team.id) + elementsHash = await this.db.createElementGroup(elements, team.id) } const insertResult = await this.db.postgresQuery( 'INSERT INTO posthog_event (created_at, event, distinct_id, properties, team_id, timestamp, elements, elements_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *', @@ -495,45 +497,6 @@ export class EventsProcessor { return data } - private async createElementGroup(elements: Element[], teamId: number): Promise { - const cleanedElements = elements.map((element, index) => ({ ...element, order: index })) - const hash = hashElements(cleanedElements) - - try { - const insertResult = await this.db.postgresQuery( - 'INSERT INTO posthog_elementgroup (hash, team_id) VALUES ($1, $2) RETURNING *', - [hash, teamId] - ) - const elementGroup = insertResult.rows[0] as ElementGroup - for (const element of cleanedElements) { - await this.db.postgresQuery( - 'INSERT INTO posthog_element (text, tag_name, href, attr_id, nth_child, nth_of_type, attributes, "order", event_id, attr_class, group_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)', - [ - element.text, - element.tag_name, - element.href, - element.attr_id, - element.nth_child, - element.nth_of_type, - element.attributes, - element.order, - element.event_id, - element.attr_class, - elementGroup.id, - ] - ) - } - } catch (error) { - // Throw further if not postgres error nr "23505" == "unique_violation" - // https://www.postgresql.org/docs/12/errcodes-appendix.html - if (error.code !== '23505') { - throw error - } - } - - return hash - } - private async createSessionRecordingEvent( uuid: string, team_id: number, @@ -542,7 +505,10 @@ export class EventsProcessor { timestamp: DateTime | string, snapshot_data: Record ): Promise { - const timestampString = castTimestampOrNow(timestamp) + const timestampString = castTimestampOrNow( + timestamp, + this.kafkaProducer ? TimestampFormat.ClickHouse : TimestampFormat.ISO + ) const data: SessionRecordingEvent = { uuid, diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index fdab780b..5b4cbb2a 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -50,7 +50,7 @@ export function elementsToString(elements: Element[]): string { ) el_string += ':' el_string += Object.entries(attributes) - .map(([key, value]) => `${key}=${value}`) + .map(([key, value]) => `${key}="${value}"`) .join('') return el_string }) @@ -109,3 +109,61 @@ export function hashElements(elements: Element[]): string { return crypto.createHash('md5').update(serializedString).digest('hex') } + +export function chainToElements(chain: string): Element[] { + const elements: Element[] = [] + + // Below splits all elements by ;, while ignoring escaped quotes and semicolons within quotes + const splitChainRegex = /(?:[^\s;"]|"(?:\\.|[^"])*")+/g + + // Below splits the tag/classes from attributes + // Needs a regex because classes can have : too + const splitClassAttributes = /(.*?)($|:([a-zA-Z\-_0-9]*=.*))/g + const parseAttributesRegex = /((.*?)="(.*?[^\\])")/gm + + Array.from(chain.matchAll(splitChainRegex)) + .map((r) => r[0]) + .forEach((elString, index) => { + const elStringSplit = Array.from(elString.matchAll(splitClassAttributes))[0] + const attributes = + elStringSplit.length > 3 + ? Array.from(elStringSplit[3].matchAll(parseAttributesRegex)).map((a) => [a[2], a[3]]) + : [] + + const element: Element = { + attributes: {}, + order: index, + } + + if (elStringSplit[1]) { + const tagAndClass = elStringSplit[1].split('.') + element.tag_name = tagAndClass[0] + if (tagAndClass.length > 1) { + const [_, ...rest] = tagAndClass + element.attr_class = rest.filter((t) => t) + } + } + + for (const [key, value] of attributes) { + if (key == 'href') { + element.href = value + } else if (key == 'nth-child') { + element.nth_child = parseInt(value) + } else if (key == 'nth-of-type') { + element.nth_of_type = parseInt(value) + } else if (key == 'text') { + element.text = value + } else if (key == 'attr_id') { + element.attr_id = value + } else if (key) { + if (!element.attributes) { + element.attributes = {} + } + element.attributes[key] = value + } + } + elements.push(element) + }) + + return elements +} diff --git a/src/server.ts b/src/server.ts index 8deedd40..4c63b893 100644 --- a/src/server.ts +++ b/src/server.ts @@ -83,10 +83,11 @@ export async function createServer( kafka = new Kafka({ clientId: `plugin-server-v${version}-${new UUIDT()}`, brokers: serverConfig.KAFKA_HOSTS.split(','), - logLevel: logLevel.NOTHING, + logLevel: logLevel.WARN, ssl: kafkaSsl, }) kafkaProducer = kafka.producer() + await kafkaProducer?.connect() } // `node-postgres` will return dates as plain JS Date objects, which will use the local timezone. @@ -149,6 +150,7 @@ export async function createServer( server.eventsProcessor = new EventsProcessor(server as PluginsServer) const closeServer = async () => { + await kafkaProducer?.disconnect() await server.redis.quit() await server.postgres.end() } @@ -177,7 +179,6 @@ export async function startPluginsServer( let pingJob: schedule.Job | undefined let statsJob: schedule.Job | undefined let piscina: Piscina | undefined - let kafkaProducer: Producer | undefined let queue: Queue | undefined let closeServer: () => Promise | undefined let stopSchedule: () => Promise | undefined @@ -200,7 +201,6 @@ export async function startPluginsServer( await stopFastifyInstance(fastifyInstance!) } await queue?.stop() - await kafkaProducer?.disconnect() await pubSub?.quit() pingJob && schedule.cancelJob(pingJob) statsJob && schedule.cancelJob(statsJob) @@ -225,8 +225,6 @@ export async function startPluginsServer( } ;[server, closeServer] = await createServer(serverConfig, null) - await server.kafkaProducer?.connect() - piscina = makePiscina(serverConfig) const processEvent = (event: PluginEvent) => { if ((piscina?.queueSize || 0) > (server?.WORKER_CONCURRENCY || 4) * (server?.WORKER_CONCURRENCY || 4)) { diff --git a/src/types.ts b/src/types.ts index 5aeb9416..66453552 100644 --- a/src/types.ts +++ b/src/types.ts @@ -189,6 +189,8 @@ export interface RawEventMessage extends BaseEventMessage { now: string /** ISO-formatted datetime. May be empty! */ sent_at: string + /** JSON-encoded number. */ + kafka_offset: string } /** Usable event message. */ @@ -237,7 +239,7 @@ export interface Element { attr_class?: string[] nth_child?: number nth_of_type?: number - attributes: Record + attributes?: Record event_id?: number order?: number group_id?: number @@ -262,6 +264,11 @@ export interface Event { created_at: string } +export interface ClickHouseEvent extends Omit { + uuid: string + elements_chain: string +} + /** Properties shared by RawPerson and Person. */ export interface BasePerson { id: number diff --git a/src/utils.ts b/src/utils.ts index 8a619069..eccf8e5e 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -147,12 +147,24 @@ for (let i = 0; i < 256; i++) { } export class UUID { - static validateString(candidate: string): void { - if (!candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i)) { + /** + * Check whether str + * + * This does not care about RFC4122, since neither does UUIDT above. + * https://stackoverflow.com/questions/7905929/how-to-test-valid-uuid-guid + */ + static validateString(candidate: any, throwOnInvalid = true): boolean { + const isValid = Boolean( + candidate && + typeof candidate === 'string' && + candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i) + ) + if (!isValid && throwOnInvalid) { throw new Error( 'String does not match format XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX (where each X is a hexadecimal character)!' ) } + return isValid } array: Uint8Array @@ -299,6 +311,10 @@ export function castTimestampOrNow( } } +export function clickHouseTimestampToISO(timestamp: string): string { + return DateTime.fromFormat(timestamp, 'yyyy-MM-dd HH:mm:ss.u', { zone: 'UTC' }).toISO() +} + export function delay(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, ms) @@ -307,5 +323,5 @@ export function delay(ms: number): Promise { /** Remove all quotes from the provided identifier to prevent SQL injection. */ export function sanitizeSqlIdentifier(unquotedIdentifier: string): string { - return '"' + unquotedIdentifier.replace(/[^\w\d_]+/g, '') + '"' + return unquotedIdentifier.replace(/[^\w\d_]+/g, '') } diff --git a/tests/clickhouse/e2e.test.ts b/tests/clickhouse/e2e.test.ts new file mode 100644 index 00000000..1bd7e7fc --- /dev/null +++ b/tests/clickhouse/e2e.test.ts @@ -0,0 +1,61 @@ +import { LogLevel, PluginsServerConfig } from '../../src/types' +import { resetTestDatabase } from '../helpers/sql' +import { startPluginsServer } from '../../src/server' +import { makePiscina } from '../../src/worker/piscina' +import { PluginsServer } from '../../src/types' +import { createPosthog, DummyPostHog } from '../../src/extensions/posthog' +import { pluginConfig39 } from '../helpers/plugins' +import { delay, UUIDT } from '../../src/utils' +import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' +import { resetKafka } from '../helpers/kafka' +import { delayUntilEventIngested } from '../shared/process-event' + +jest.setTimeout(60000) // 60 sec timeout + +const extraServerConfig: Partial = { + KAFKA_ENABLED: true, + KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', + WORKER_CONCURRENCY: 2, + PLUGIN_SERVER_INGESTION: true, + LOG_LEVEL: LogLevel.Log, +} + +describe('e2e clickhouse ingestion', () => { + let server: PluginsServer + let stopServer: () => Promise + let posthog: DummyPostHog + + beforeAll(async () => { + await resetKafka(extraServerConfig) + }) + + beforeEach(async () => { + await resetTestDatabase(` + async function processEvent (event) { + event.properties.processed = 'hell yes' + event.properties.upperUuid = event.properties.uuid?.toUpperCase() + return event + } + `) + await resetTestDatabaseClickhouse(extraServerConfig) + const startResponse = await startPluginsServer(extraServerConfig, makePiscina) + server = startResponse.server + stopServer = startResponse.stop + posthog = createPosthog(server, pluginConfig39) + }) + + afterEach(async () => { + await stopServer() + }) + + test('event captured, processed, ingested', async () => { + expect((await server.db.fetchEvents()).length).toBe(0) + const uuid = new UUIDT().toString() + posthog.capture('custom event', { name: 'haha', uuid }) + await delayUntilEventIngested(() => server.db.fetchEvents()) + const events = await server.db.fetchEvents() + expect(events.length).toBe(1) + expect(events[0].properties.processed).toEqual('hell yes') + expect(events[0].properties.upperUuid).toEqual(uuid.toUpperCase()) + }) +}) diff --git a/tests/clickhouse/ingestion-utils.test.ts b/tests/clickhouse/ingestion-utils.test.ts new file mode 100644 index 00000000..ee2ea40f --- /dev/null +++ b/tests/clickhouse/ingestion-utils.test.ts @@ -0,0 +1,50 @@ +import { chainToElements, elementsToString } from '../../src/ingestion/utils' + +test('elementsToString and chainToElements', async () => { + const elementsString = elementsToString([ + { + tag_name: 'a', + href: '/a-url', + attr_class: ['small'], + text: 'bla bla', + attributes: { + prop: 'value', + number: 33, + 'data-attr': 'something " that; could mess up', + style: 'min-height: 100vh;', + }, + nth_child: 1, + nth_of_type: 0, + }, + { tag_name: 'button', attr_class: ['btn', 'btn-primary'], nth_child: 0, nth_of_type: 0 }, + { tag_name: 'div', nth_child: 0, nth_of_type: 0 }, + { tag_name: 'div', nth_child: 0, nth_of_type: 0, attr_id: 'nested' }, + ]) + + expect(elementsString).toEqual( + [ + 'a.small:data-attr="something \\" that; could mess up"href="/a-url"nth-child="1"nth-of-type="0"number="33"prop="value"style="min-height: 100vh;"text="bla bla"', + 'button.btn.btn-primary:nth-child="0"nth-of-type="0"', + 'div:nth-child="0"nth-of-type="0"', + 'div:attr_id="nested"nth-child="0"nth-of-type="0"', + ].join(';') + ) + + const elements = chainToElements(elementsString) + expect(elements.length).toBe(4) + expect(elements[0].tag_name).toEqual('a') + expect(elements[0].href).toEqual('/a-url') + expect(elements[0].attr_class).toEqual(['small']) + expect(elements[0].attributes).toEqual({ + prop: 'value', + number: '33', + // NB! The original Python code also does not unescape `\"` -> `"` + // Could be fixed later, but keeping as is for parity. + 'data-attr': 'something \\" that; could mess up', + style: 'min-height: 100vh;', + }) + expect(elements[0].nth_child).toEqual(1) + expect(elements[0].nth_of_type).toEqual(0) + expect(elements[1].attr_class).toEqual(['btn', 'btn-primary']) + expect(elements[3].attr_id).toEqual('nested') +}) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 96214920..7604e29d 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -1,66 +1,23 @@ -import { PluginsServerConfig } from '../../src/types' +import { PluginsServerConfig, Event } from '../../src/types' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' -import { KafkaCollector, KafkaObserver } from '../helpers/kafka' -import { UUIDT } from '../../src/utils' -import { DateTime } from 'luxon' +import { resetKafka } from '../helpers/kafka' import { createProcessEventTests } from '../shared/process-event' jest.setTimeout(180_000) // 3 minute timeout const extraServerConfig: Partial = { KAFKA_ENABLED: true, - KAFKA_HOSTS: 'kafka:9092', - DATABASE_URL: 'postgres://posthog:posthog@localhost:5439/test_posthog', + KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', } describe('process event (clickhouse)', () => { - const kafkaObserver = new KafkaObserver(extraServerConfig) + beforeAll(async () => { + await resetKafka(extraServerConfig) + }) beforeEach(async () => { await resetTestDatabaseClickhouse(extraServerConfig) }) - const server = createProcessEventTests( - 'clickhouse', - { - getSessionRecordingEvents: (server) => server.db.fetchSessionRecordingEvents(), - getEvents: (server) => server.db.fetchEvents(), - getPersons: (server) => server.db.fetchPersons(), - getDistinctIds: (server, person) => server.db.fetchDistinctIdValues(person), - getElements: (server) => server.db.fetchElements(), - }, - extraServerConfig - ) - - test('event is passed through', async () => { - const uuid = new UUIDT().toString() - const now = DateTime.utc() - console.log('starting kafka observer') - await kafkaObserver.start() - console.log('sending message') - const kafkaCollector = new KafkaCollector(kafkaObserver) - await kafkaObserver.handOffMessage({ - distinct_id: 'abcd', - ip: '1.1.1.1', - site_url: 'x.com', - team_id: 1, - uuid, - data: { - distinct_id: 'abcd', - ip: '1.1.1.1', - site_url: 'x.com', - team_id: 1, - now: now.toString(), - event: 'test', - uuid, - }, - now, - sent_at: null, - }) - console.log('waiting for messages') - const processedMessages = await kafkaCollector.collect(1) - - console.log(processedMessages) - expect(1).toEqual(1) - }) + createProcessEventTests('clickhouse', extraServerConfig) }) diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 15613922..5195e85c 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -1,9 +1,13 @@ import { EventEmitter } from 'events' import { Kafka, Consumer, logLevel, EachMessagePayload, Producer } from 'kafkajs' -import { KAFKA_EVENTS, KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' +import { + KAFKA_EVENTS, + KAFKA_EVENTS_INGESTION_HANDOFF, + KAFKA_SESSION_RECORDING_EVENTS, +} from '../../src/ingestion/topics' import { parseRawEventMessage } from '../../src/ingestion/utils' import { EventMessage, PluginsServerConfig } from '../../src/types' -import { UUIDT } from '../../src/utils' +import { delay, UUIDT } from '../../src/utils' import { defaultConfig, overrideWithEnv } from '../../src/config' export class KafkaObserver extends EventEmitter { @@ -19,7 +23,7 @@ export class KafkaObserver extends EventEmitter { this.kafka = new Kafka({ clientId: `plugin-server-test-${new UUIDT()}`, brokers: (config.KAFKA_HOSTS || '').split(','), - logLevel: logLevel.NOTHING, + logLevel: logLevel.WARN, }) this.producer = this.kafka.producer() this.consumer = this.kafka.consumer({ @@ -107,3 +111,76 @@ export class KafkaCollector extends EventEmitter { }) } } + +/** Clear the kafka queue */ +export async function resetKafka(extraServerConfig: Partial, delayMs = 2000) { + console.log('Resetting Kafka!') + const config = { ...overrideWithEnv(defaultConfig, process.env), ...extraServerConfig } + const kafka = new Kafka({ + clientId: `plugin-server-test-${new UUIDT()}`, + brokers: (config.KAFKA_HOSTS || '').split(','), + logLevel: logLevel.WARN, + }) + const producer = kafka.producer() + const consumer = kafka.consumer({ + groupId: 'clickhouse-ingestion-test', + }) + const messages = [] + + async function createTopic(topic: string) { + try { + const admin = kafka.admin() + await admin.connect() + await admin.createTopics({ + waitForLeaders: true, + topics: [{ topic }], + }) + await admin.disconnect() + } catch (e) { + if (!e?.error?.includes('Topic with this name already exists')) { + console.error(`Error creating kafka topic "${topic}". This might be fine!`) + console.error(e) + } + } + } + + await createTopic(KAFKA_EVENTS_INGESTION_HANDOFF) + await createTopic(KAFKA_SESSION_RECORDING_EVENTS) + + const connected = await new Promise(async (resolve, reject) => { + console.info('setting group join and crash listeners') + const { CONNECT, GROUP_JOIN, CRASH } = consumer.events + consumer.on(CONNECT, () => { + console.log('consumer connected to kafka') + }) + consumer.on(GROUP_JOIN, () => { + console.log('joined group') + resolve() + }) + consumer.on(CRASH, ({ payload: { error } }) => reject(error)) + console.info('connecting producer') + await producer.connect() + console.info('subscribing consumer') + + await consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) + console.info('running consumer') + await consumer.run({ + eachMessage: async (payload) => { + console.info('message received!') + messages.push(payload) + }, + }) + }) + + console.info(`awaiting ${delayMs} ms before disconnecting`) + await delay(delayMs) + + console.info('disconnecting producer') + await producer.disconnect() + console.info('stopping consumer') + await consumer.stop() + console.info('disconnecting consumer') + await consumer.disconnect() + + return true +} diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 92e39a95..4c1aef36 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -81,6 +81,7 @@ export async function createUserTeamAndOrganization( created_at: new Date().toISOString(), updated_at: new Date().toISOString(), personalization: '{}', + setup_section_2_completed: true, }) await insertRow(db, 'posthog_organizationmembership', { id: organizationMembershipId, diff --git a/tests/postgres/e2e.test.ts b/tests/postgres/e2e.test.ts index 208eef5a..dc3a2b18 100644 --- a/tests/postgres/e2e.test.ts +++ b/tests/postgres/e2e.test.ts @@ -5,7 +5,8 @@ import { makePiscina } from '../../src/worker/piscina' import { PluginsServer } from '../../src/types' import { createPosthog, DummyPostHog } from '../../src/extensions/posthog' import { pluginConfig39 } from '../helpers/plugins' -import { delay } from '../../src/utils' +import { delay, UUIDT } from '../../src/utils' +import { delayUntilEventIngested } from '../shared/process-event' jest.setTimeout(60000) // 60 sec timeout @@ -18,6 +19,7 @@ describe('e2e postgres ingestion', () => { await resetTestDatabase(` async function processEvent (event) { event.properties.processed = 'hell yes' + event.properties.upperUuid = event.properties.uuid?.toUpperCase() return event } `) @@ -47,10 +49,12 @@ describe('e2e postgres ingestion', () => { test('event captured, processed, ingested', async () => { expect((await server.db.fetchEvents()).length).toBe(0) - posthog.capture('custom event', { name: 'haha' }) - await delay(2000) + const uuid = new UUIDT().toString() + posthog.capture('custom event', { name: 'haha', uuid, randomProperty: 'lololo' }) + await delayUntilEventIngested(() => server.db.fetchEvents()) const events = await server.db.fetchEvents() expect(events.length).toBe(1) expect(events[0].properties.processed).toEqual('hell yes') + expect(events[0].properties.upperUuid).toEqual(uuid.toUpperCase()) }) }) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index d404d826..2c12142b 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,14 +1,54 @@ -import { PluginsServer, Event, Person, PersonDistinctId, Element, PostgresSessionRecordingEvent } from '../../src/types' import { createProcessEventTests } from '../shared/process-event' +import { createUserTeamAndOrganization } from '../helpers/sql' +import { Team } from '../../src/types' jest.setTimeout(600000) // 600 sec timeout describe('process event (postgresql)', () => { - createProcessEventTests('postgresql', { - getSessionRecordingEvents: (server) => server.db.fetchSessionRecordingEvents(), - getEvents: (server) => server.db.fetchEvents(), - getPersons: (server) => server.db.fetchPersons(), - getDistinctIds: (server, person) => server.db.fetchDistinctIdValues(person), - getElements: (server) => server.db.fetchElements(), + createProcessEventTests('postgresql', {}, (response) => { + test('element group', async () => { + const { server } = response + const elements = [{ tag_name: 'button', text: 'Sign up!' }, { tag_name: 'div' }] + + const elementsHash = server!.db.createElementGroup(elements, 2) + const elementGroup = await server!.db.fetchElements() + + expect(elementGroup[0].tag_name).toEqual('button') + expect(elementGroup[1].tag_name).toEqual('div') + expect(elementGroup.length).toEqual(2) + + const elements2 = [ + { tag_name: 'button', text: 'Sign up!' }, + // make sure we remove events if we can + { tag_name: 'div', event: { id: 'blabla' } }, + ] + + const elementsHash2 = server!.db.createElementGroup(elements2, 2) + const elementGroup2 = await server!.db.fetchElements() + // we are fetching all the elements, so expect there to be no new ones + expect(elementGroup2.length).toEqual(2) + expect(elementsHash).toEqual(elementsHash2) + + await createUserTeamAndOrganization( + server!.postgres, + 3, + 1002, + '01774e2f-0d01-0000-ee94-9a238640c6ee', + '0174f81e-36f5-0000-7ef8-cc26c1fbab1c' + ) + + const teams = (await server!.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows as Team[] + + // # Test no team leakage + const team2 = teams[1] + + const elementsHash3 = server!.db.createElementGroup(elements2, 3) + const elementGroup3 = await server!.db.fetchElements() + console.log(elementGroup3) + // created new elements as it's different team even if the hash is the same + expect(elementGroup3.length).toEqual(4) + expect(elementsHash).toEqual(elementsHash2) + expect(elementsHash).toEqual(elementsHash3) + }) }) }) diff --git a/tests/shared/process-event.ts b/tests/shared/process-event.ts index 1417dc52..b9a4c058 100644 --- a/tests/shared/process-event.ts +++ b/tests/shared/process-event.ts @@ -9,11 +9,15 @@ import { Element, PostgresSessionRecordingEvent, PluginsServerConfig, + ClickHouseEvent, + SessionRecordingEvent, } from '../../src/types' import { createUserTeamAndOrganization, resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' -import { UUIDT } from '../../src/utils' +import { delay, UUIDT } from '../../src/utils' +import { IEvent } from '../../src/idl/protos' +import { hashElements } from '../../src/ingestion/utils' jest.setTimeout(600000) // 600 sec timeout @@ -25,6 +29,15 @@ async function getFirstTeam(server: PluginsServer): Promise { return (await getTeams(server))[0] } +export async function delayUntilEventIngested(fetchEvents: () => Promise, minCount = 1): Promise { + for (let i = 0; i < 30; i++) { + if ((await fetchEvents()).length >= minCount) { + return + } + await delay(500) + } +} + async function createPerson( server: PluginsServer, team: Team, @@ -34,29 +47,21 @@ async function createPerson( return server.db.createPerson(DateTime.utc(), properties, team.id, null, false, new UUIDT().toString(), distinctIds) } +type ReturnWithServer = { server?: PluginsServer; stopServer?: () => Promise } + export const createProcessEventTests = ( database: 'postgresql' | 'clickhouse', - { - getSessionRecordingEvents, - getEvents, - getPersons, - getDistinctIds, - getElements, - }: { - getSessionRecordingEvents: (server: PluginsServer) => Promise - getEvents: (server: PluginsServer) => Promise - getPersons: (server: PluginsServer) => Promise - getDistinctIds: (server: PluginsServer, person: Person) => Promise - getElements: (server: PluginsServer, event: Event) => Promise - }, - extraServerConfig?: Partial -): PluginsServer => { + extraServerConfig?: Partial, + createTests?: (response: ReturnWithServer) => void +): ReturnWithServer => { let queryCounter = 0 + let processEventCounter = 0 let team: Team let server: PluginsServer let stopServer: () => Promise let eventsProcessor: EventsProcessor let now = DateTime.utc() + const returned: ReturnWithServer = {} async function getServer(): Promise<[PluginsServer, () => Promise]> { const [server, stopServer] = await createServer({ @@ -78,6 +83,32 @@ export const createProcessEventTests = ( return [server, stopServer] } + async function processEvent( + distinctId: string, + ip: string, + siteUrl: string, + data: PluginEvent, + teamId: number, + now: DateTime, + sentAt: DateTime | null, + eventUuid: string + ): Promise { + const response = await eventsProcessor.processEvent( + distinctId, + ip, + siteUrl, + data, + teamId, + now, + sentAt, + eventUuid + ) + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchEvents(), ++processEventCounter) + } + return response + } + beforeEach(async () => { const testCode = ` function processEvent (event, meta) { @@ -87,8 +118,11 @@ export const createProcessEventTests = ( ` await resetTestDatabase(testCode, extraServerConfig) ;[server, stopServer] = await getServer() + returned.server = server + returned.stopServer = stopServer eventsProcessor = new EventsProcessor(server) queryCounter = 0 + processEventCounter = 0 team = await getFirstTeam(server) now = DateTime.utc() }) @@ -97,13 +131,15 @@ export const createProcessEventTests = ( await stopServer?.() }) + createTests?.(returned) + test('capture new person', async () => { await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [true, team.id]) team = await getFirstTeam(server) expect(team.event_names).toEqual([]) - await eventsProcessor.processEvent( + await processEvent( '2', '', '', @@ -124,18 +160,14 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect(queryCounter).toBe(12) - - // TODO: 12 vs 28 is a big difference. Why so? - // num_queries = 28 - // if settings.EE_AVAILABLE: # extra queries to check for hooks - // num_queries += 4 - // if settings.MULTI_TENANCY: # extra query to check for billing plan - // num_queries += 1 - // with self.assertNumQueries(num_queries): + if (database === 'clickhouse') { + expect(queryCounter).toBe(8) + } else if (database === 'postgresql') { + expect(queryCounter).toBe(12) + } // capture a second time to verify e.g. event_names is not ['$autocapture', '$autocapture'] - await eventsProcessor.processEvent( + await processEvent( '2', '', '', @@ -156,26 +188,31 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const events = await getEvents(server) - const persons = await getPersons(server) + const events = await server.db.fetchEvents() + const persons = await server.db.fetchPersons() expect(events.length).toEqual(2) expect(persons.length).toEqual(1) - const [event] = events const [person] = persons - const distinctIds = await getDistinctIds(server, person) + const distinctIds = await server.db.fetchDistinctIdValues(person) + const [event] = events as Event[] expect(event.distinct_id).toEqual('2') expect(distinctIds).toEqual(['2']) expect(event.event).toEqual('$autocapture') - expect(event.elements_hash).toEqual('0679137c0cd2408a2906839143e7a71f') - const elements = await getElements(server, event) + const elements = await server.db.fetchElements(event) expect(elements[0].tag_name).toEqual('a') expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) expect(elements[1].order).toEqual(1) expect(elements[1].text).toEqual('๐Ÿ’ป') + if (database === 'clickhouse') { + expect(hashElements(elements)).toEqual('0679137c0cd2408a2906839143e7a71f') + } else if (database === 'postgresql') { + expect(event.elements_hash).toEqual('0679137c0cd2408a2906839143e7a71f') + } + team = await getFirstTeam(server) expect(team.event_names).toEqual(['$autocapture']) expect(team.event_names_with_usage).toEqual([{ event: '$autocapture', volume: null, usage_count: null }]) @@ -190,7 +227,7 @@ export const createProcessEventTests = ( test('capture no element', async () => { await createPerson(server, team, ['asdfasdfasdf']) - await eventsProcessor.processEvent( + await processEvent( 'asdfasdfasdf', '', '', @@ -204,8 +241,8 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual(['asdfasdfasdf']) - const [event] = await getEvents(server) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual(['asdfasdfasdf']) + const [event] = await server.db.fetchEvents() expect(event.event).toBe('$pageview') }) @@ -216,7 +253,7 @@ export const createProcessEventTests = ( const tomorrow = rightNow.plus({ days: 1, hours: 2 }) const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }) - await eventsProcessor.processEvent( + await processEvent( 'movie played', '', '', @@ -231,7 +268,7 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds expect(eventSecondsBeforeNow).toBeGreaterThan(590) @@ -249,7 +286,7 @@ export const createProcessEventTests = ( // tomorrow = tomorrow.replace(tzinfo=None) // tomorrow_sent_at = tomorrow_sent_at.replace(tzinfo=None) - await eventsProcessor.processEvent( + await processEvent( 'movie played', '', '', @@ -264,7 +301,7 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds expect(eventSecondsBeforeNow).toBeGreaterThan(590) @@ -277,7 +314,7 @@ export const createProcessEventTests = ( const rightNow = DateTime.utc() const tomorrow = rightNow.plus({ days: 1, hours: 2 }) - await eventsProcessor.processEvent( + await processEvent( 'movie played', '', '', @@ -292,7 +329,7 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() const difference = tomorrow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds expect(difference).toBeLessThan(1) }) @@ -300,7 +337,7 @@ export const createProcessEventTests = ( test('ip capture', async () => { await createPerson(server, team, ['asdfasdfasdf']) - await eventsProcessor.processEvent( + await processEvent( 'asdfasdfasdf', '11.12.13.14', '', @@ -313,14 +350,14 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.properties['$ip']).toBe('11.12.13.14') }) test('ip override', async () => { await createPerson(server, team, ['asdfasdfasdf']) - await eventsProcessor.processEvent( + await processEvent( 'asdfasdfasdf', '11.12.13.14', '', @@ -333,7 +370,8 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - const [event] = await getEvents(server) + + const [event] = await server.db.fetchEvents() expect(event.properties['$ip']).toBe('1.0.0.1') }) @@ -341,7 +379,7 @@ export const createProcessEventTests = ( await server.db.postgresQuery('update posthog_team set anonymize_ips = $1', [true]) await createPerson(server, team, ['asdfasdfasdf']) - await eventsProcessor.processEvent( + await processEvent( 'asdfasdfasdf', '11.12.13.14', '', @@ -354,14 +392,15 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - const [event] = await getEvents(server) + + const [event] = await server.db.fetchEvents() expect(event.properties['$ip']).not.toBeDefined() }) test('alias', async () => { await createPerson(server, team, ['old_distinct_id']) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -375,8 +414,8 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) - expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ 'old_distinct_id', 'new_distinct_id', ]) @@ -385,7 +424,7 @@ export const createProcessEventTests = ( test('alias reverse', async () => { await createPerson(server, team, ['old_distinct_id']) - await eventsProcessor.processEvent( + await processEvent( 'old_distinct_id', '', '', @@ -399,8 +438,8 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) - expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ 'old_distinct_id', 'new_distinct_id', ]) @@ -409,7 +448,7 @@ export const createProcessEventTests = ( test('alias twice', async () => { await createPerson(server, team, ['old_distinct_id']) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -423,17 +462,17 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getPersons(server)).length).toBe(1) - expect((await getEvents(server)).length).toBe(1) - expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + expect((await server.db.fetchPersons()).length).toBe(1) + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ 'old_distinct_id', 'new_distinct_id', ]) await createPerson(server, team, ['old_distinct_id_2']) - expect((await getPersons(server)).length).toBe(2) + expect((await server.db.fetchPersons()).length).toBe(2) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -446,9 +485,9 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(2) - expect((await getPersons(server)).length).toBe(1) - expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + expect((await server.db.fetchEvents()).length).toBe(2) + expect((await server.db.fetchPersons()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ 'old_distinct_id', 'new_distinct_id', 'old_distinct_id_2', @@ -456,7 +495,7 @@ export const createProcessEventTests = ( }) test('alias before person', async () => { - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -470,9 +509,9 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) - expect((await getPersons(server)).length).toBe(1) - expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + expect((await server.db.fetchEvents()).length).toBe(1) + expect((await server.db.fetchPersons()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ 'new_distinct_id', 'old_distinct_id', ]) @@ -482,7 +521,7 @@ export const createProcessEventTests = ( await createPerson(server, team, ['old_distinct_id']) await createPerson(server, team, ['new_distinct_id']) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -496,8 +535,8 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) - expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ 'old_distinct_id', 'new_distinct_id', ]) @@ -506,7 +545,7 @@ export const createProcessEventTests = ( test('offset timestamp', async () => { now = DateTime.fromISO('2020-01-01T12:00:05.200Z') - await eventsProcessor.processEvent( + await processEvent( 'distinct_id', '', '', @@ -516,16 +555,16 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) + expect((await server.db.fetchEvents()).length).toBe(1) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') }) test('offset timestamp no sent_at', async () => { now = DateTime.fromISO('2020-01-01T12:00:05.200Z') - await eventsProcessor.processEvent( + await processEvent( 'distinct_id', '', '', @@ -535,9 +574,9 @@ export const createProcessEventTests = ( null, new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) + expect((await server.db.fetchEvents()).length).toBe(1) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') }) @@ -551,7 +590,7 @@ export const createProcessEventTests = ( key_on_new: 'new value', }) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -565,10 +604,10 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) - expect((await getPersons(server)).length).toBe(1) - const [person] = await getPersons(server) - expect(await getDistinctIds(server, person)).toEqual(['old_distinct_id', 'new_distinct_id']) + expect((await server.db.fetchEvents()).length).toBe(1) + expect((await server.db.fetchPersons()).length).toBe(1) + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['old_distinct_id', 'new_distinct_id']) expect(person.properties).toEqual({ key_on_both: 'new value both', key_on_new: 'new value', @@ -577,7 +616,7 @@ export const createProcessEventTests = ( }) test('long htext', async () => { - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -604,11 +643,15 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const [event] = await getEvents(server) - expect(event.elements_hash).toEqual('c2659b28e72835706835764cf7f63c2a') - const [element] = await getElements(server, event) + const [event] = (await server.db.fetchEvents()) as Event[] + const [element] = await server.db.fetchElements(event) expect(element.href?.length).toEqual(2048) expect(element.text?.length).toEqual(400) + if (database === 'postgresql') { + expect(event.elements_hash).toEqual('c2659b28e72835706835764cf7f63c2a') + } else if (database === 'clickhouse') { + expect(hashElements([element])).toEqual('c2659b28e72835706835764cf7f63c2a') + } }) test('capture first team event', async () => { @@ -619,7 +662,7 @@ export const createProcessEventTests = ( capture: jest.fn((event, properties) => true), } as any - await eventsProcessor.processEvent( + await processEvent( '2', '', '', @@ -645,8 +688,13 @@ export const createProcessEventTests = ( team = await getFirstTeam(server) expect(team.ingested_event).toEqual(true) - const [event] = await getEvents(server) - expect(event.elements_hash).toEqual('a89021a60b3497d24e93ae181fba01aa') + const [event] = (await server.db.fetchEvents()) as Event[] + if (database === 'postgresql') { + expect(event.elements_hash).toEqual('a89021a60b3497d24e93ae181fba01aa') + } else if (database === 'clickhouse') { + const elements = await server.db.fetchElements(event) + expect(hashElements(elements)).toEqual('a89021a60b3497d24e93ae181fba01aa') + } }) test('snapshot event stored as session_recording_event', async () => { @@ -663,11 +711,12 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) + await delayUntilEventIngested(() => server.db.fetchSessionRecordingEvents()) - const events = await getEvents(server) + const events = await server.db.fetchEvents() expect(events.length).toEqual(0) - const sessionRecordingEvents = await getSessionRecordingEvents(server) + const sessionRecordingEvents = await server.db.fetchSessionRecordingEvents() expect(sessionRecordingEvents.length).toBe(1) const [event] = sessionRecordingEvents @@ -679,7 +728,7 @@ export const createProcessEventTests = ( test('identify set', async () => { await createPerson(server, team, ['distinct_id']) - await eventsProcessor.processEvent( + await processEvent( 'distinct_id', '', '', @@ -697,17 +746,17 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) + expect((await server.db.fetchEvents()).length).toBe(1) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.properties['$set']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) - const [person] = await getPersons(server) - expect(await getDistinctIds(server, person)).toEqual(['distinct_id']) + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['distinct_id']) expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) expect(person.is_identified).toEqual(true) - await eventsProcessor.processEvent( + await processEvent( 'distinct_id', '', '', @@ -724,15 +773,15 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(2) - const [person2] = await getPersons(server) + expect((await server.db.fetchEvents()).length).toBe(2) + const [person2] = await server.db.fetchPersons() expect(person2.properties).toEqual({ a_prop: 'test-2', b_prop: 'test-2b', c_prop: 'test-1' }) }) test('identify set_once', async () => { await createPerson(server, team, ['distinct_id']) - await eventsProcessor.processEvent( + await processEvent( 'distinct_id', '', '', @@ -750,17 +799,17 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) + expect((await server.db.fetchEvents()).length).toBe(1) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.properties['$set_once']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) - const [person] = await getPersons(server) - expect(await getDistinctIds(server, person)).toEqual(['distinct_id']) + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['distinct_id']) expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) expect(person.is_identified).toEqual(true) - await eventsProcessor.processEvent( + await processEvent( 'distinct_id', '', '', @@ -777,15 +826,15 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(2) - const [person2] = await getPersons(server) + expect((await server.db.fetchEvents()).length).toBe(2) + const [person2] = await server.db.fetchPersons() expect(person2.properties).toEqual({ a_prop: 'test-1', b_prop: 'test-2b', c_prop: 'test-1' }) }) test('distinct with anonymous_id', async () => { await createPerson(server, team, ['anonymous_id']) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -804,15 +853,15 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - expect((await getEvents(server)).length).toBe(1) - const [event] = await getEvents(server) + expect((await server.db.fetchEvents()).length).toBe(1) + const [event] = await server.db.fetchEvents() expect(event.properties['$set']).toEqual({ a_prop: 'test' }) - const [person] = await getPersons(server) - expect(await getDistinctIds(server, person)).toEqual(['anonymous_id', 'new_distinct_id']) + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['anonymous_id', 'new_distinct_id']) expect(person.properties).toEqual({ a_prop: 'test' }) // check no errors as this call can happen multiple times - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -841,7 +890,7 @@ export const createProcessEventTests = ( await createPerson(server, team, ['anonymous_id']) await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -859,8 +908,8 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const [person] = await getPersons(server) - expect(await getDistinctIds(server, person)).toEqual(['anonymous_id', 'new_distinct_id']) + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['anonymous_id', 'new_distinct_id']) expect(person.properties['email']).toEqual('someone@gmail.com') }) @@ -868,7 +917,7 @@ export const createProcessEventTests = ( await createPerson(server, team, ['anonymous_id']) await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -886,14 +935,14 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const persons1 = await getPersons(server) + const persons1 = await server.db.fetchPersons() expect(persons1.length).toBe(1) - expect(await getDistinctIds(server, persons1[0])).toEqual(['anonymous_id', 'new_distinct_id']) + expect(await server.db.fetchDistinctIdValues(persons1[0])).toEqual(['anonymous_id', 'new_distinct_id']) expect(persons1[0].properties['email']).toEqual('someone@gmail.com') await createPerson(server, team, ['anonymous_id_2']) - await eventsProcessor.processEvent( + await processEvent( 'new_distinct_id', '', '', @@ -911,9 +960,13 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const persons2 = await getPersons(server) + const persons2 = await server.db.fetchPersons() expect(persons2.length).toBe(1) - expect(await getDistinctIds(server, persons2[0])).toEqual(['anonymous_id', 'new_distinct_id', 'anonymous_id_2']) + expect(await server.db.fetchDistinctIdValues(persons2[0])).toEqual([ + 'anonymous_id', + 'new_distinct_id', + 'anonymous_id_2', + ]) expect(persons2[0].properties['email']).toEqual('someone@gmail.com') }) @@ -929,7 +982,7 @@ export const createProcessEventTests = ( await createPerson(server, team2, ['2'], { email: 'team2@gmail.com' }) await createPerson(server, team, ['1', '2']) - await eventsProcessor.processEvent( + await processEvent( '2', '', '', @@ -947,13 +1000,13 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const people = await getPersons(server) + const people = await server.db.fetchPersons() expect(people.length).toEqual(2) expect(people[1].team_id).toEqual(team.id) expect(people[1].properties).toEqual({}) - expect(await getDistinctIds(server, people[1])).toEqual(['1', '2']) + expect(await server.db.fetchDistinctIdValues(people[1])).toEqual(['1', '2']) expect(people[0].team_id).toEqual(team2.id) - expect(await getDistinctIds(server, people[0])).toEqual(['2']) + expect(await server.db.fetchDistinctIdValues(people[0])).toEqual(['2']) }) test('set is_identified', async () => { @@ -961,7 +1014,7 @@ export const createProcessEventTests = ( const person1 = await createPerson(server, team, [distinct_id]) expect(person1.is_identified).toBe(false) - await eventsProcessor.processEvent( + await processEvent( distinct_id, '', '', @@ -972,14 +1025,14 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - const [person2] = await getPersons(server) + const [person2] = await server.db.fetchPersons() expect(person2.is_identified).toBe(true) }) test('team event_properties', async () => { expect(team.event_properties_numerical).toEqual([]) - await eventsProcessor.processEvent( + await processEvent( 'xxx', '', '', @@ -996,7 +1049,7 @@ export const createProcessEventTests = ( }) test('event name object json', async () => { - await eventsProcessor.processEvent( + await processEvent( 'xxx', '', '', @@ -1006,12 +1059,12 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.event).toEqual('{"event name":"as object"}') }) test('event name array json', async () => { - await eventsProcessor.processEvent( + await processEvent( 'xxx', '', '', @@ -1021,12 +1074,12 @@ export const createProcessEventTests = ( now, new UUIDT().toString() ) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.event).toEqual('["event name","a list"]') }) test('long event name substr', async () => { - await eventsProcessor.processEvent( + await processEvent( 'xxx', '', '', @@ -1034,12 +1087,40 @@ export const createProcessEventTests = ( team.id, DateTime.utc(), DateTime.utc(), - 'uuid' + new UUIDT().toString() ) - const [event] = await getEvents(server) + const [event] = await server.db.fetchEvents() expect(event.event?.length).toBe(200) }) - return server! + test('throws with bad uuid', async () => { + await expect( + processEvent( + 'xxx', + '', + '', + ({ event: 'E', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + 'this is not an uuid' + ) + ).rejects.toEqual(new Error('Not a valid UUID: "this is not an uuid"')) + + await expect( + processEvent( + 'xxx', + '', + '', + ({ event: 'E', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + null as any + ) + ).rejects.toEqual(new Error('Not a valid UUID: "null"')) + }) + + return returned } diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 0331b53a..27e1de0a 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -303,6 +303,6 @@ describe('sanitizeSqlIdentifier', () => { const sanitizedIdentifier = sanitizeSqlIdentifier(rawIdentifier) - expect(sanitizedIdentifier).toStrictEqual('"some_fieldDROPTABLEactually_an_injection9"') + expect(sanitizedIdentifier).toStrictEqual('some_fieldDROPTABLEactually_an_injection9') }) }) From 345c275de7f17010f82ef14fb7306e06bfd97189 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 3 Feb 2021 01:15:05 +0100 Subject: [PATCH 208/218] Update README.md with PLUGIN_SERVER_INGESTION --- README.md | 61 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 40f65124..631e88a6 100644 --- a/README.md +++ b/README.md @@ -27,36 +27,37 @@ Let's get you developing the plugin server in no time: There's a multitude of settings you can use to control the plugin server. Use them as environment variables. -| Name | Description | Default value | -| ----------------------------- | ---------------------------------------------------------- | ------------------------------------- | -| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | -| REDIS_URL | Redis store URL | `'redis://localhost'` | -| BASE_DIR | base path for resolving local plugins | `'.'` | -| WORKER_CONCURRENCY | number of concurrent worker threads | `0` โ€“ all cores | -| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | -| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | -| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | -| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | -| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | -| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | -| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | -| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | -| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | -| CLICKHOUSE_CA | ClickHouse CA certs | `null` | -| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | -| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | -| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | -| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | -| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | -| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | -| DISABLE_WEB | whether to disable web server | `true` | -| WEB_PORT | port for web server to listen on | `3008` | -| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | -| LOG_LEVEL | minimum log level | `LogLevel.Info` | -| SENTRY_DSN | Sentry ingestion URL | `null` | -| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | -| STATSD_PORT | StatsD port | `8125` | -| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | +| Name | Description | Default value | +| ----------------------------- | ------------------------------------------------------------------- | ------------------------------------- | +| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | +| REDIS_URL | Redis store URL | `'redis://localhost'` | +| BASE_DIR | base path for resolving local plugins | `'.'` | +| WORKER_CONCURRENCY | number of concurrent worker threads | `0` โ€“ all cores | +| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | +| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | +| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | +| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | +| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | +| PLUGIN_SERVER_INGESTION | Whether the plugin server should put events right into the database | `false` | +| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | +| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | +| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | +| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | +| CLICKHOUSE_CA | ClickHouse CA certs | `null` | +| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | +| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | +| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | +| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | +| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | +| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | +| DISABLE_WEB | whether to disable web server | `true` | +| WEB_PORT | port for web server to listen on | `3008` | +| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | +| LOG_LEVEL | minimum log level | `LogLevel.Info` | +| SENTRY_DSN | Sentry ingestion URL | `null` | +| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | +| STATSD_PORT | StatsD port | `8125` | +| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | ## Questions? From 15903a9d8ef1b429462da4ece93defe515d18abb Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 3 Feb 2021 12:28:37 +0100 Subject: [PATCH 209/218] rename CLICKHOUSE_USERNAME --> CLICKHOUSE_USER for consistency with the rest of the app --- README.md | 2 +- src/config.ts | 4 ++-- src/server.ts | 2 +- src/types.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 631e88a6..a566067d 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ There's a multitude of settings you can use to control the plugin server. Use th | PLUGIN_SERVER_INGESTION | Whether the plugin server should put events right into the database | `false` | | CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | | CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | -| CLICKHOUSE_USERNAME | ClickHouse username | `'default'` | +| CLICKHOUSE_USER | ClickHouse username | `'default'` | | CLICKHOUSE_PASSWORD | ClickHouse password | `null` | | CLICKHOUSE_CA | ClickHouse CA certs | `null` | | CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | diff --git a/src/config.ts b/src/config.ts index a1758dc4..d4846552 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,7 +11,7 @@ export function getDefaultConfig(): PluginsServerConfig { DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/test_posthog' : 'postgres://localhost:5432/posthog', CLICKHOUSE_HOST: 'localhost', CLICKHOUSE_DATABASE: isTestEnv ? 'posthog_test' : 'default', - CLICKHOUSE_USERNAME: 'default', + CLICKHOUSE_USER: 'default', CLICKHOUSE_PASSWORD: null, CLICKHOUSE_CA: null, CLICKHOUSE_SECURE: false, @@ -47,7 +47,7 @@ export function getConfigHelp(): Record { DATABASE_URL: 'Postgres database URL', CLICKHOUSE_HOST: 'ClickHouse host', CLICKHOUSE_DATABASE: 'ClickHouse database', - CLICKHOUSE_USERNAME: 'ClickHouse username', + CLICKHOUSE_USER: 'ClickHouse username', CLICKHOUSE_PASSWORD: 'ClickHouse password', CLICKHOUSE_CA: 'ClickHouse CA certs', CLICKHOUSE_SECURE: 'Secure ClickHouse connection', diff --git a/src/server.ts b/src/server.ts index 4c63b893..2ff92100 100644 --- a/src/server.ts +++ b/src/server.ts @@ -73,7 +73,7 @@ export async function createServer( url: `http${serverConfig.CLICKHOUSE_SECURE ? 's' : ''}://$${serverConfig.CLICKHOUSE_HOST}`, port: serverConfig.CLICKHOUSE_SECURE ? 8443 : 8123, basicAuth: { - username: serverConfig.CLICKHOUSE_USERNAME, + username: serverConfig.CLICKHOUSE_USER, password: serverConfig.CLICKHOUSE_PASSWORD, }, config: { diff --git a/src/types.ts b/src/types.ts index 66453552..97bfef79 100644 --- a/src/types.ts +++ b/src/types.ts @@ -25,7 +25,7 @@ export interface PluginsServerConfig extends Record { DATABASE_URL: string CLICKHOUSE_HOST: string CLICKHOUSE_DATABASE: string - CLICKHOUSE_USERNAME: string + CLICKHOUSE_USER: string CLICKHOUSE_PASSWORD: string | null CLICKHOUSE_CA: string | null CLICKHOUSE_SECURE: boolean From bde20a7ab560eb4cfff817359294efd495a7fe77 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Wed, 3 Feb 2021 16:09:27 +0100 Subject: [PATCH 210/218] Delete run-tests-locally.sh --- run-tests-locally.sh | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100755 run-tests-locally.sh diff --git a/run-tests-locally.sh b/run-tests-locally.sh deleted file mode 100755 index 1839b938..00000000 --- a/run-tests-locally.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set +ex - -if [[ -z $PRIMARY_DB ]]; then - echo 'You must provide var PRIMARY_DB! Either "postgres" or "clickhouse"' - exit 1 -fi - -export REDIS_URL='redis://localhost' -export CLICKHOUSE_HOST=localhost -export CLICKHOUSE_DATABASE=posthog_test -export KAFKA_ENABLED=true -export KAFKA_HOSTS=kafka:9092 - -source ../posthog/env/bin/activate -SECRET_KEY=abc DATABASE_URL='postgres://postgres:postgres@localhost:5432/posthog' TEST=true python ../posthog/manage.py setup_test_environment -DATABASE_URL='postgres://postgres:postgres@localhost:5432/test_posthog' yarn test:$PRIMARY_DB From b15c61ad764faeaf15cc58bd0e1eb5ddcffccd2d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 4 Feb 2021 08:45:35 +0100 Subject: [PATCH 211/218] Resolve all kafka offsets (#124) * resolve all kafka offsets * remove a few lines * clean up * sort events according to offsets * add sort info * simplify offset line * add comment * Type pluginEvents as PluginEvent[] * Commit last offset of batch manually Co-authored-by: Michael Matloka --- src/ingestion/kafka-queue.ts | 48 +++++++++++++++++++----------------- src/plugins.ts | 2 +- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 26d1cc69..8e8273d3 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -6,14 +6,12 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' import { killGracefully } from '../utils' -export type BatchCallback = (messages: Message[]) => Promise - export class KafkaQueue implements Queue { private pluginsServer: PluginsServer private kafka: Kafka private consumer: Consumer private wasConsumerRan: boolean - private processEventBatch: (batch: PluginEvent[]) => Promise + private processEventBatch: (batch: PluginEvent[]) => Promise private saveEvent: (event: PluginEvent) => Promise constructor( @@ -34,32 +32,33 @@ export class KafkaQueue implements Queue { resolveOffset, heartbeat, commitOffsetsIfNecessary, - uncommittedOffsets, isRunning, isStale, }: EachBatchPayload): Promise { const batchProcessingTimer = new Date() - const rawEvents: RawEventMessage[] = batch.messages.map((message) => ({ - ...JSON.parse(message.value!.toString()), - kafka_offset: message.offset, - })) - const parsedEvents = rawEvents.map((rawEvent) => ({ - ...rawEvent, - data: JSON.parse(rawEvent.data), - })) - const pluginEvents: PluginEvent[] = rawEvents.map((rawEvent) => { - const { data: dataStr, ...restOfRawEvent } = rawEvent - const event = { ...restOfRawEvent, ...JSON.parse(dataStr) } + + const uuidOrder = new Map() + const uuidOffset = new Map() + const pluginEvents: PluginEvent[] = batch.messages.map((message, index) => { + const { data: dataStr, ...rawEvent } = JSON.parse(message.value!.toString()) + const event = { ...rawEvent, ...JSON.parse(dataStr) } + uuidOrder.set(event.uuid, index) + uuidOffset.set(event.uuid, message.offset) return { ...event, - kafka_offset: restOfRawEvent.kafka_offset, site_url: event.site_url || null, ip: event.ip || null, } }) - const processedEvents: PluginEvent[] = ( - await this.processEventBatch(pluginEvents) - ).filter((event: PluginEvent[] | false | null | undefined) => Boolean(event)) + + const processedEvents = await this.processEventBatch(pluginEvents) + + // Sort in the original order that the events came in, putting any randomly added events to the end. + // This is so we would resolve the correct kafka offsets in order. + processedEvents.sort( + (a, b) => (uuidOrder.get(a.uuid!) || pluginEvents.length) - (uuidOrder.get(b.uuid!) || pluginEvents.length) + ) + for (const event of processedEvents) { if (!isRunning()) { status.info('๐Ÿ˜ฎ', 'Consumer not running anymore, canceling batch processing!') @@ -71,12 +70,17 @@ export class KafkaQueue implements Queue { } const singleIngestionTimer = new Date() await this.saveEvent(event) - resolveOffset(event.kafka_offset!) + const offset = uuidOffset.get(event.uuid!) + if (offset) { + resolveOffset(offset) + } await heartbeat() await commitOffsetsIfNecessary() this.pluginsServer.statsd?.timing('kafka_queue.single_ingestion', singleIngestionTimer) } this.pluginsServer.statsd?.timing('kafka_queue.each_batch', batchProcessingTimer) + resolveOffset(batch.lastOffset()) + await commitOffsetsIfNecessary() } async start(): Promise { @@ -88,9 +92,7 @@ export class KafkaQueue implements Queue { await this.consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) // KafkaJS batching: https://kafka.js.org/docs/consuming#a-name-each-batch-a-eachbatch await this.consumer.run({ - // TODO: eachBatchAutoResolve: false, // don't autoresolve whole batch in case we exit it early - // The issue is right now we'd miss some messages and not resolve them as processEventBatch COMPETELY - // discards some events, leaving us with no kafka_offset to resolve when in fact it should be resolved. + eachBatchAutoResolve: false, // we are resolving the last offset of the batch more deliberately autoCommitInterval: 500, // autocommit every 500 msโ€ฆ autoCommitThreshold: 1000, // โ€ฆor every 1000 messages, whichever is sooner eachBatch: this.eachBatch.bind(this), diff --git a/src/plugins.ts b/src/plugins.ts index 95ffb91e..b80976e8 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -264,7 +264,7 @@ export async function runPluginsOnBatch(server: PluginsServer, batch: PluginEven allReturnedEvents = allReturnedEvents.concat(returnedEvents) } - return allReturnedEvents + return allReturnedEvents.filter(Boolean) } export async function runPluginTask(server: PluginsServer, taskName: string, pluginConfigId: number): Promise { From 8001e2c39f6cf45d1ab486db92faaed533f3797e Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 4 Feb 2021 08:51:13 +0100 Subject: [PATCH 212/218] Add KAFKA_INCOMING_TOPIC and clean up unused code (#125) * ingest to clickhouse only if PLUGIN_SERVER_INGESTION is enabled * KAFKA_INCOMING_TOPIC * test clickhouse connection * remove some unused code * Replace _INCOMING_ with _CONSUMPTION_ * Add KAFKA_CONSUMPTION_TOPIC to README.md * add // Co-authored-by: Michael Matloka --- README.md | 1 + src/config.ts | 2 + src/extensions/posthog.ts | 3 +- src/ingestion/kafka-queue.ts | 9 +- src/ingestion/utils.ts | 16 +--- src/server.ts | 10 +++ src/types.ts | 1 + src/worker/queue.ts | 22 ++--- tests/clickhouse/e2e.test.ts | 2 + tests/clickhouse/process-event.test.ts | 3 + tests/helpers/kafka.ts | 114 +------------------------ 11 files changed, 40 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index a566067d..6d1e254b 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ There's a multitude of settings you can use to control the plugin server. Use th | CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | | KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | | KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | +| KAFKA_CONSUMPTION_TOPIC | Kafka consumption topic override | `null` (automatic) | | KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | | KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | | KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | diff --git a/src/config.ts b/src/config.ts index d4846552..f0888db2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,6 +20,7 @@ export function getDefaultConfig(): PluginsServerConfig { KAFKA_CLIENT_CERT_B64: null, KAFKA_CLIENT_CERT_KEY_B64: null, KAFKA_TRUSTED_CERT_B64: null, + KAFKA_CONSUMPTION_TOPIC: null, PLUGIN_SERVER_INGESTION: false, PLUGINS_CELERY_QUEUE: 'posthog-plugins', REDIS_URL: 'redis://127.0.0.1', @@ -62,6 +63,7 @@ export function getConfigHelp(): Record { LOG_LEVEL: 'minimum log level', KAFKA_ENABLED: 'use Kafka instead of Celery to ingest events', KAFKA_HOSTS: 'comma-delimited Kafka hosts', + KAFKA_CONSUMPTION_TOPIC: 'Kafka consumption topic override', KAFKA_CLIENT_CERT_B64: 'Kafka certificate in Base64', KAFKA_CLIENT_CERT_KEY_B64: 'Kafka certificate key in Base64', KAFKA_TRUSTED_CERT_B64: 'Kafka trusted CA in Base64', diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index 94af08ca..863d662d 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -1,4 +1,3 @@ -import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../ingestion/topics' import { Properties } from '@posthog/plugin-scaffold' import { DateTime } from 'luxon' import { PluginsServer, PluginConfig, RawEventMessage } from 'types' @@ -31,7 +30,7 @@ export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig) throw new Error('kafkaProducer not configured!') } server.kafkaProducer.send({ - topic: KAFKA_EVENTS_INGESTION_HANDOFF, + topic: server.KAFKA_CONSUMPTION_TOPIC!, messages: [ { key: data.uuid, diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 8e8273d3..7c1ba2a9 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -1,7 +1,6 @@ import * as Sentry from '@sentry/node' import { Kafka, Consumer, Message, EachBatchPayload } from 'kafkajs' import { PluginsServer, Queue, RawEventMessage } from 'types' -import { KAFKA_EVENTS_INGESTION_HANDOFF } from './topics' import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' import { killGracefully } from '../utils' @@ -89,7 +88,7 @@ export class KafkaQueue implements Queue { this.consumer.on(this.consumer.events.CRASH, ({ payload: { error } }) => reject(error)) status.info('โฌ', `Connecting Kafka consumer to ${this.pluginsServer.KAFKA_HOSTS}...`) this.wasConsumerRan = true - await this.consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) + await this.consumer.subscribe({ topic: this.pluginsServer.KAFKA_CONSUMPTION_TOPIC! }) // KafkaJS batching: https://kafka.js.org/docs/consuming#a-name-each-batch-a-eachbatch await this.consumer.run({ eachBatchAutoResolve: false, // we are resolving the last offset of the batch more deliberately @@ -106,7 +105,7 @@ export class KafkaQueue implements Queue { return } status.info('โณ', 'Pausing Kafka consumer...') - await this.consumer.pause([{ topic: KAFKA_EVENTS_INGESTION_HANDOFF }]) + await this.consumer.pause([{ topic: this.pluginsServer.KAFKA_CONSUMPTION_TOPIC! }]) status.info('โธ', 'Kafka consumer paused!') } @@ -115,12 +114,12 @@ export class KafkaQueue implements Queue { return } status.info('โณ', 'Resuming Kafka consumer...') - await this.consumer.resume([{ topic: KAFKA_EVENTS_INGESTION_HANDOFF }]) + await this.consumer.resume([{ topic: this.pluginsServer.KAFKA_CONSUMPTION_TOPIC! }]) status.info('โ–ถ๏ธ', 'Kafka consumer resumed!') } isPaused(): boolean { - return this.consumer.paused().some(({ topic }) => topic === KAFKA_EVENTS_INGESTION_HANDOFF) + return this.consumer.paused().some(({ topic }) => topic === this.pluginsServer.KAFKA_CONSUMPTION_TOPIC) } async stop(): Promise { diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 5b4cbb2a..ba46ab3b 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -1,20 +1,6 @@ -import { DateTime } from 'luxon' -import { Element, BaseEventMessage, RawEventMessage, EventMessage, BasePerson, RawPerson, Person } from '../types' +import { Element, BasePerson, RawPerson, Person } from '../types' import crypto from 'crypto' -export function parseRawEventMessage(message: RawEventMessage): EventMessage { - return { - ...(message as BaseEventMessage), - data: JSON.parse(message.data), - now: DateTime.fromISO(message.now), - sent_at: DateTime.fromISO(message.sent_at), - } -} - -export function parseRawPerson(rawPerson: RawPerson): Person { - return { ...(rawPerson as BasePerson), created_at: DateTime.fromISO(rawPerson.created_at) } -} - export function unparsePersonPartial(person: Partial): Partial { return { ...(person as BasePerson), ...(person.created_at ? { created_at: person.created_at.toISO() } : {}) } } diff --git a/src/server.ts b/src/server.ts index 2ff92100..9442ce31 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,6 +20,7 @@ import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' import { DateTime } from 'luxon' +import { KAFKA_EVENTS_INGESTION_HANDOFF, KAFKA_EVENTS_WAL } from './ingestion/topics' export async function createServer( config: Partial = {}, @@ -80,6 +81,15 @@ export async function createServer( database: serverConfig.CLICKHOUSE_DATABASE, }, }) + await clickhouse.query('SELECT 1') // test that the connection works + + if (!serverConfig.KAFKA_CONSUMPTION_TOPIC) { + // When ingesting events, listen to the "INGESTION_HANDOFF" topic, otherwise listen to the "WAL" and discard + serverConfig.KAFKA_CONSUMPTION_TOPIC = serverConfig.PLUGIN_SERVER_INGESTION + ? KAFKA_EVENTS_INGESTION_HANDOFF + : KAFKA_EVENTS_WAL + } + kafka = new Kafka({ clientId: `plugin-server-v${version}-${new UUIDT()}`, brokers: serverConfig.KAFKA_HOSTS.split(','), diff --git a/src/types.ts b/src/types.ts index 97bfef79..5e644da9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -34,6 +34,7 @@ export interface PluginsServerConfig extends Record { KAFKA_CLIENT_CERT_B64: string | null KAFKA_CLIENT_CERT_KEY_B64: string | null KAFKA_TRUSTED_CERT_B64: string | null + KAFKA_CONSUMPTION_TOPIC: string | null PLUGINS_CELERY_QUEUE: string REDIS_URL: string BASE_DIR: string diff --git a/src/worker/queue.ts b/src/worker/queue.ts index f55f1995..4cf943ab 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -92,16 +92,18 @@ async function startQueueKafka( status.error('โ“', 'UUID missing in event received from Kafka!') return } - await server.eventsProcessor.processEvent( - distinct_id, - ip, - site_url, - event, - team_id, - DateTime.fromISO(now), - sent_at ? DateTime.fromISO(sent_at) : null, - uuid - ) + if (server.PLUGIN_SERVER_INGESTION) { + await server.eventsProcessor.processEvent( + distinct_id, + ip, + site_url, + event, + team_id, + DateTime.fromISO(now), + sent_at ? DateTime.fromISO(sent_at) : null, + uuid + ) + } }) await kafkaQueue.start() diff --git a/tests/clickhouse/e2e.test.ts b/tests/clickhouse/e2e.test.ts index 1bd7e7fc..51ac85f2 100644 --- a/tests/clickhouse/e2e.test.ts +++ b/tests/clickhouse/e2e.test.ts @@ -9,6 +9,7 @@ import { delay, UUIDT } from '../../src/utils' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { resetKafka } from '../helpers/kafka' import { delayUntilEventIngested } from '../shared/process-event' +import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' jest.setTimeout(60000) // 60 sec timeout @@ -17,6 +18,7 @@ const extraServerConfig: Partial = { KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', WORKER_CONCURRENCY: 2, PLUGIN_SERVER_INGESTION: true, + KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_INGESTION_HANDOFF, LOG_LEVEL: LogLevel.Log, } diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 7604e29d..41167ddf 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -2,12 +2,15 @@ import { PluginsServerConfig, Event } from '../../src/types' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { resetKafka } from '../helpers/kafka' import { createProcessEventTests } from '../shared/process-event' +import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' jest.setTimeout(180_000) // 3 minute timeout const extraServerConfig: Partial = { KAFKA_ENABLED: true, KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', + PLUGIN_SERVER_INGESTION: true, + KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_INGESTION_HANDOFF, } describe('process event (clickhouse)', () => { diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 5195e85c..7d1b535a 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -1,116 +1,8 @@ -import { EventEmitter } from 'events' -import { Kafka, Consumer, logLevel, EachMessagePayload, Producer } from 'kafkajs' -import { - KAFKA_EVENTS, - KAFKA_EVENTS_INGESTION_HANDOFF, - KAFKA_SESSION_RECORDING_EVENTS, -} from '../../src/ingestion/topics' -import { parseRawEventMessage } from '../../src/ingestion/utils' -import { EventMessage, PluginsServerConfig } from '../../src/types' +import { Kafka, logLevel } from 'kafkajs' +import { PluginsServerConfig } from '../../src/types' import { delay, UUIDT } from '../../src/utils' import { defaultConfig, overrideWithEnv } from '../../src/config' - -export class KafkaObserver extends EventEmitter { - public kafka: Kafka - public producer: Producer - public consumer: Consumer - - private isStarted: boolean - - constructor(extraServerConfig: Partial) { - super() - const config = { ...overrideWithEnv(defaultConfig, process.env), ...extraServerConfig } - this.kafka = new Kafka({ - clientId: `plugin-server-test-${new UUIDT()}`, - brokers: (config.KAFKA_HOSTS || '').split(','), - logLevel: logLevel.WARN, - }) - this.producer = this.kafka.producer() - this.consumer = this.kafka.consumer({ - groupId: 'clickhouse-ingestion-test', - }) - this.isStarted = false - } - - public async start(): Promise { - console.info('observer started!') - if (this.isStarted) { - return - } - this.isStarted = true - return await new Promise(async (resolve, reject) => { - console.info('connecting producer') - await this.producer.connect() - console.info('subscribing consumer') - await this.consumer.subscribe({ topic: KAFKA_EVENTS }) - console.info('running consumer') - await this.consumer.run({ - eachMessage: async (payload) => { - console.info('message received!') - this.emit('message', payload) - }, - }) - console.info('setting group join and crash listeners') - const { CONNECT, GROUP_JOIN, CRASH } = this.consumer.events - this.consumer.on(CONNECT, () => { - console.log('consumer connected to kafka') - }) - this.consumer.on(GROUP_JOIN, () => { - console.log('joined group') - resolve() - }) - this.consumer.on(CRASH, ({ payload: { error } }) => reject(error)) - }) - } - - public async stop(): Promise { - this.removeAllListeners() - console.info('disconnecting producer') - await this.producer.disconnect() - console.info('stopping consumer') - await this.consumer.stop() - console.info('disconnecting consumer') - await this.consumer.disconnect() - } - - public async handOffMessage(message: EventMessage): Promise { - console.info('producing message') - await this.producer.send({ - topic: KAFKA_EVENTS_INGESTION_HANDOFF, - messages: [{ value: Buffer.from(JSON.stringify(message)) }], - }) - } -} - -export class KafkaCollector extends EventEmitter { - collection: EventMessage[] - kafkaObserver: KafkaObserver - - constructor(kafkaObserver: KafkaObserver) { - super() - this.collection = [] - this.kafkaObserver = kafkaObserver - kafkaObserver.addListener('message', (payload: EachMessagePayload) => { - console.info('message received') - this.collection.push(parseRawEventMessage(JSON.parse(payload.message.value!.toString()))) - this.emit('message') - }) - } - - async collect(numberOfMessages: number): Promise { - return await new Promise((resolve) => { - const resolveIfCollectedEnough = () => { - console.log('collection:', this.collection) - if (this.collection.length >= numberOfMessages) { - this.removeListener('message', resolveIfCollectedEnough) - resolve(this.collection) - } - } - this.addListener('message', resolveIfCollectedEnough) - resolveIfCollectedEnough() - }) - } -} +import { KAFKA_EVENTS_INGESTION_HANDOFF, KAFKA_SESSION_RECORDING_EVENTS } from '../../src/ingestion/topics' /** Clear the kafka queue */ export async function resetKafka(extraServerConfig: Partial, delayMs = 2000) { From bec6bfb85578dc49ec40c22416a70c97d13f1ec0 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 4 Feb 2021 12:26:51 +0100 Subject: [PATCH 213/218] Postgres parity tests (#126) * create postgres parity tests, fix some bugs * create all topics * person and distinct id parity tests * add TODOs * fetch distinct ids from clickhouse * create a specialised function for moving distinct ids and fix postgres/clickhouse person_id difference (number vs string) * test for updating distinct ids * createPerson, updatePerson and deletePerson * add a debug line to help debug flaky github action * remove falsehood --- src/db.ts | 152 ++++++++---- src/ingestion/process-event.ts | 8 +- src/types.ts | 23 ++ src/utils.ts | 7 + tests/clickhouse/postgres-parity.test.ts | 298 +++++++++++++++++++++++ tests/helpers/kafka.ts | 46 ++-- tests/helpers/sql.ts | 10 +- tests/postgres/process-event.test.ts | 2 + tests/shared/process-event.ts | 10 +- tests/utils.test.ts | 11 + 10 files changed, 482 insertions(+), 85 deletions(-) create mode 100644 tests/clickhouse/postgres-parity.test.ts diff --git a/src/db.ts b/src/db.ts index 98f635dc..4458298b 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,5 +1,5 @@ import { Properties } from '@posthog/plugin-scaffold' -import { ClickHouse, QueryCursor } from 'clickhouse' +import { ClickHouse } from 'clickhouse' import { Producer } from 'kafkajs' import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' @@ -7,18 +7,22 @@ import { string } from 'yargs' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { chainToElements, hashElements, unparsePersonPartial } from './ingestion/utils' import { + ClickHouseEvent, + ClickHousePerson, + ClickHousePersonDistinctId, + Database, + Element, + ElementGroup, + Event, Person, PersonDistinctId, - RawPerson, - RawOrganization, PostgresSessionRecordingEvent, - Event, - ClickHouseEvent, - Element, + RawOrganization, + RawPerson, SessionRecordingEvent, - ElementGroup, + TimestampFormat, } from './types' -import { castTimestampOrNow, clickHouseTimestampToISO, sanitizeSqlIdentifier } from './utils' +import { castTimestampOrNow, clickHouseTimestampToISO, escapeClickHouseString, sanitizeSqlIdentifier } from './utils' /** The recommended way of accessing the database. */ export class DB { @@ -53,9 +57,22 @@ export class DB { // Person - public async fetchPersons(): Promise { - const result = await this.postgresQuery('SELECT * FROM posthog_person') - return result.rows as Person[] + public async fetchPersons(database?: Database.Postgres): Promise + public async fetchPersons(database: Database.ClickHouse): Promise + public async fetchPersons(database: Database = Database.Postgres): Promise { + if (database === Database.ClickHouse) { + return (await this.clickhouseQuery('SELECT * FROM person')) as ClickHousePerson[] + } else if (database === Database.Postgres) { + return ((await this.postgresQuery('SELECT * FROM posthog_person')).rows as RawPerson[]).map( + (rawPerson: RawPerson) => + ({ + ...rawPerson, + created_at: DateTime.fromISO(rawPerson.created_at).toUTC(), + } as Person) + ) + } else { + throw new Error(`Can't fetch persons for database: ${database}`) + } } public async fetchPerson(teamId: number, distinctId: string): Promise { @@ -75,7 +92,7 @@ export class DB { ) if (selectResult.rows.length > 0) { const rawPerson: RawPerson = selectResult.rows[0] - return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } + return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at).toUTC() } } } @@ -92,10 +109,11 @@ export class DB { 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', [createdAt.toISO(), JSON.stringify(properties), teamId, isUserId, isIdentified, uuid] ) - const personCreated = insertResult.rows[0] as Person + const personCreated = insertResult.rows[0] as RawPerson + const person = { ...personCreated, created_at: DateTime.fromISO(personCreated.created_at).toUTC() } as Person if (this.kafkaProducer) { const data = { - created_at: castTimestampOrNow(createdAt), + created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouse), properties: JSON.stringify(properties), team_id: teamId, is_identified: isIdentified, @@ -108,10 +126,9 @@ export class DB { } for (const distinctId of distinctIds || []) { - await this.addDistinctId(personCreated, distinctId) + await this.addDistinctId(person, distinctId) } - - return personCreated + return person } public async updatePerson(person: Person, update: Partial): Promise { @@ -123,39 +140,71 @@ export class DB { )} WHERE id = $${Object.values(update).length + 1}`, values ) - if (this.kafkaProducer) { - const data = { - created_at: castTimestampOrNow(updatedPerson.created_at), - properties: JSON.stringify(updatedPerson.properties), - team_id: updatedPerson.team_id, - is_identified: updatedPerson.is_identified, - id: updatedPerson.uuid.toString(), + if (this.clickhouse) { + const { is_user_id, id, uuid, ...validUpdates } = update + const updateString = Object.entries(validUpdates) + .map(([key, value]) => { + let clickhouseValue: string + if (typeof value === 'string') { + clickhouseValue = value + } else if (typeof value === 'boolean') { + clickhouseValue = value ? '1' : '0' + } else if (DateTime.isDateTime(value)) { + clickhouseValue = castTimestampOrNow(value, TimestampFormat.ClickHouse) + } else { + clickhouseValue = JSON.stringify(value) + } + return `${sanitizeSqlIdentifier(key)} = '${escapeClickHouseString(clickhouseValue)}'` + }) + .join(', ') + if (updateString.length > 0) { + await this.clickhouseQuery( + `ALTER TABLE person UPDATE ${updateString} WHERE id = '${escapeClickHouseString(person.uuid)}'` + ) } - await this.kafkaProducer.send({ - topic: KAFKA_PERSON, - messages: [{ value: Buffer.from(JSON.stringify(data)) }], - }) } return updatedPerson } - public async deletePerson(personId: number): Promise { - await this.postgresQuery('DELETE FROM posthog_persondistinctid WHERE person_id = $1', [personId]) - await this.postgresQuery('DELETE FROM posthog_person WHERE id = $1', [personId]) + public async deletePerson(person: Person): Promise { + await this.postgresQuery('DELETE FROM posthog_persondistinctid WHERE person_id = $1', [person.id]) + await this.postgresQuery('DELETE FROM posthog_person WHERE id = $1', [person.id]) if (this.clickhouse) { - await this.clickhouseQuery(`ALTER TABLE person DELETE WHERE id = ${personId}`) - await this.clickhouseQuery(`ALTER TABLE person_distinct_id DELETE WHERE person_id = ${personId}`) + await this.clickhouseQuery(`ALTER TABLE person DELETE WHERE id = '${escapeClickHouseString(person.uuid)}'`) + await this.clickhouseQuery( + `ALTER TABLE person_distinct_id DELETE WHERE person_id = '${escapeClickHouseString(person.uuid)}'` + ) } } // PersonDistinctId - public async fetchDistinctIdValues(person: Person): Promise { - const result = await this.postgresQuery( - 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', - [person.id, person.team_id] - ) - return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) + public async fetchDistinctIds(person: Person, database?: Database.Postgres): Promise + public async fetchDistinctIds(person: Person, database: Database.ClickHouse): Promise + public async fetchDistinctIds( + person: Person, + database: Database = Database.Postgres + ): Promise { + if (database === Database.ClickHouse) { + return (await this.clickhouseQuery( + `SELECT * FROM person_distinct_id WHERE person_id='${escapeClickHouseString( + person.uuid + )}' and team_id='${person.team_id}' ORDER BY id` + )) as ClickHousePersonDistinctId[] + } else if (database === Database.Postgres) { + const result = await this.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', + [person.id, person.team_id] + ) + return result.rows as PersonDistinctId[] + } else { + throw new Error(`Can't fetch persons for database: ${database}`) + } + } + + public async fetchDistinctIdValues(person: Person, database: Database = Database.Postgres): Promise { + const personDistinctIds = await this.fetchDistinctIds(person, database as any) + return personDistinctIds.map((pdi) => pdi.distinct_id) } public async addDistinctId(person: Person, distinctId: string): Promise { @@ -167,26 +216,27 @@ export class DB { if (this.kafkaProducer) { await this.kafkaProducer.send({ topic: KAFKA_PERSON_UNIQUE_ID, - messages: [{ value: Buffer.from(JSON.stringify(personDistinctIdCreated)) }], + messages: [ + { value: Buffer.from(JSON.stringify({ ...personDistinctIdCreated, person_id: person.uuid })) }, + ], }) } } - public async updateDistinctId( + public async moveDistinctId( + person: Person, personDistinctId: PersonDistinctId, - update: Partial + moveToPerson: Person ): Promise { - const updatedPersonDistinctId: PersonDistinctId = { ...personDistinctId, ...update } - await this.postgresQuery( - `UPDATE posthog_persondistinctid SET ${Object.keys(update).map( - (field, index) => `"${sanitizeSqlIdentifier(field)}" = $${index + 1}` - )} WHERE id = $${Object.values(update).length + 1}`, - [...Object.values(update), personDistinctId.id] - ) + await this.postgresQuery(`UPDATE posthog_persondistinctid SET person_id = $1 WHERE id = $2`, [ + moveToPerson.id, + personDistinctId.id, + ]) if (this.kafkaProducer) { + const clickhouseModel: ClickHousePersonDistinctId = { ...personDistinctId, person_id: moveToPerson.uuid } await this.kafkaProducer.send({ topic: KAFKA_PERSON_UNIQUE_ID, - messages: [{ value: Buffer.from(JSON.stringify(updatedPersonDistinctId)) }], + messages: [{ value: Buffer.from(JSON.stringify(clickhouseModel)) }], }) } } @@ -248,7 +298,7 @@ export class DB { public async fetchElements(event?: Event): Promise { if (this.kafkaProducer) { const events = (await this.clickhouseQuery( - `SELECT elements_chain FROM events WHERE uuid='${sanitizeSqlIdentifier((event as any).uuid)}'` + `SELECT elements_chain FROM events WHERE uuid='${escapeClickHouseString((event as any).uuid)}'` )) as ClickHouseEvent[] const chain = events?.[0]?.elements_chain return chainToElements(chain) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 304fd9d2..acc584e6 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -288,7 +288,7 @@ export class EventsProcessor { ) ).rows for (const personDistinctId of otherPersonDistinctIds) { - await this.db.updateDistinctId(personDistinctId, { person_id: mergeInto.id }) + await this.db.moveDistinctId(otherPerson, personDistinctId, mergeInto) } const otherCohortPeople: CohortPeople[] = ( @@ -301,7 +301,7 @@ export class EventsProcessor { ]) } - await this.db.deletePerson(otherPerson.id) + await this.db.deletePerson(otherPerson) } } @@ -359,9 +359,9 @@ export class EventsProcessor { teamId, null, false, - personUuid.toString() + personUuid.toString(), + [distinctId] ) - await this.db.addDistinctId(personCreated, distinctId) } catch {} } diff --git a/src/types.ts b/src/types.ts index 5e644da9..4e70ef5c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -290,6 +290,16 @@ export interface Person extends BasePerson { created_at: DateTime } +/** Clickhouse Person model. */ +export interface ClickHousePerson { + id: string + created_at: string + team_id: number + properties: string + is_identified: number + timestamp: string +} + /** Usable PersonDistinctId model. */ export interface PersonDistinctId { id: number @@ -298,6 +308,14 @@ export interface PersonDistinctId { distinct_id: string } +/** ClickHouse PersonDistinctId model. */ +export interface ClickHousePersonDistinctId { + id: number + team_id: number + person_id: string + distinct_id: string +} + /** Usable CohortPeople model. */ export interface CohortPeople { id: number @@ -323,3 +341,8 @@ export enum TimestampFormat { ClickHouse = 'clickhouse', ISO = 'iso', } + +export enum Database { + ClickHouse = 'clickhouse', + Postgres = 'postgres', +} diff --git a/src/utils.ts b/src/utils.ts index eccf8e5e..bd6227a4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -325,3 +325,10 @@ export function delay(ms: number): Promise { export function sanitizeSqlIdentifier(unquotedIdentifier: string): string { return unquotedIdentifier.replace(/[^\w\d_]+/g, '') } + +/** Escape single quotes and slashes */ +export function escapeClickHouseString(string: string): string { + // In string literals, you need to escape at least `'` and `\`. + // https://clickhouse.tech/docs/en/sql-reference/syntax/ + return string.replace(/\\/g, '\\\\').replace(/'/g, "\\'") +} diff --git a/tests/clickhouse/postgres-parity.test.ts b/tests/clickhouse/postgres-parity.test.ts new file mode 100644 index 00000000..a964faf1 --- /dev/null +++ b/tests/clickhouse/postgres-parity.test.ts @@ -0,0 +1,298 @@ +import { Database, LogLevel, PluginsServer, PluginsServerConfig, Team, TimestampFormat } from '../../src/types' +import { getFirstTeam, resetTestDatabase } from '../helpers/sql' +import { startPluginsServer } from '../../src/server' +import { makePiscina } from '../../src/worker/piscina' +import { createPosthog, DummyPostHog } from '../../src/extensions/posthog' +import { pluginConfig39 } from '../helpers/plugins' +import { castTimestampOrNow, UUIDT } from '../../src/utils' +import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' +import { resetKafka } from '../helpers/kafka' +import { delayUntilEventIngested } from '../shared/process-event' +import { DateTime } from 'luxon' + +jest.setTimeout(60000) // 60 sec timeout + +const extraServerConfig: Partial = { + KAFKA_ENABLED: true, + KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', + WORKER_CONCURRENCY: 2, + PLUGIN_SERVER_INGESTION: true, + LOG_LEVEL: LogLevel.Log, +} + +describe('postgres parity', () => { + let server: PluginsServer + let stopServer: () => Promise + let posthog: DummyPostHog + let team: Team + + beforeAll(async () => { + await resetKafka(extraServerConfig) + }) + + beforeEach(async () => { + await resetTestDatabase(` + async function processEvent (event) { + event.properties.processed = 'hell yes' + event.properties.upperUuid = event.properties.uuid?.toUpperCase() + return event + } + `) + await resetTestDatabaseClickhouse(extraServerConfig) + const startResponse = await startPluginsServer(extraServerConfig, makePiscina) + server = startResponse.server + stopServer = startResponse.stop + posthog = createPosthog(server, pluginConfig39) + team = await getFirstTeam(server) + }) + + afterEach(async () => { + await stopServer() + }) + + test('createPerson', async () => { + const uuid = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + true, + uuid, + ['distinct1', 'distinct2'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(person, Database.ClickHouse), 2) + + const clickHousePersons = await server.db.fetchPersons(Database.ClickHouse) + expect(clickHousePersons).toEqual([ + { + id: uuid, + created_at: expect.any(String), // '2021-02-04 00:18:26.472', + team_id: team.id, + properties: '{"userProp":"propValue"}', + is_identified: 1, + _timestamp: expect.any(String), + _offset: expect.any(Number), + }, + ]) + const clickHouseDistinctIds = await server.db.fetchDistinctIdValues(person, Database.ClickHouse) + expect(clickHouseDistinctIds).toEqual(['distinct1', 'distinct2']) + + const postgresPersons = await server.db.fetchPersons(Database.Postgres) + expect(postgresPersons).toEqual([ + { + id: expect.any(Number), + created_at: expect.any(DateTime), + properties: { + userProp: 'propValue', + }, + team_id: 2, + is_user_id: null, + is_identified: true, + uuid: uuid, + }, + ]) + const postgresDistinctIds = await server.db.fetchDistinctIdValues(person, Database.Postgres) + expect(postgresDistinctIds).toEqual(['distinct1', 'distinct2']) + + expect(person).toEqual(postgresPersons[0]) + }) + + test('updatePerson', async () => { + const uuid = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + false, + uuid, + ['distinct1', 'distinct2'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(person, Database.ClickHouse), 2) + + // update JSON and boolean to true + + await server.db.updatePerson(person, { properties: { replacedUserProp: 'propValue' }, is_identified: true }) + + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).filter((p) => p.is_identified) + ) + + const clickHousePersons = await server.db.fetchPersons(Database.ClickHouse) + const postgresPersons = await server.db.fetchPersons(Database.Postgres) + + expect(clickHousePersons.length).toEqual(1) + expect(postgresPersons.length).toEqual(1) + + expect(postgresPersons[0].is_identified).toEqual(true) + expect(postgresPersons[0].properties).toEqual({ replacedUserProp: 'propValue' }) + + expect(clickHousePersons[0].is_identified).toEqual(1) + expect(clickHousePersons[0].properties).toEqual('{"replacedUserProp":"propValue"}') + + // update date and boolean to false + + const randomDate = DateTime.utc().minus(100000).setZone('UTC') + await server.db.updatePerson(person, { created_at: randomDate, is_identified: false }) + + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).filter((p) => p.is_identified) + ) + + const clickHousePersons2 = await server.db.fetchPersons(Database.ClickHouse) + const postgresPersons2 = await server.db.fetchPersons(Database.Postgres) + + expect(clickHousePersons2.length).toEqual(1) + expect(postgresPersons2.length).toEqual(1) + + expect(postgresPersons2[0].is_identified).toEqual(false) + expect(postgresPersons2[0].created_at.toISO()).toEqual(randomDate.toISO()) + + expect(clickHousePersons2[0].is_identified).toEqual(0) + expect(clickHousePersons2[0].created_at).toEqual(castTimestampOrNow(randomDate, TimestampFormat.ClickHouse)) + }) + + test('deletePerson', async () => { + const uuid = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + false, + uuid, + ['distinct1', 'distinct2'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(person, Database.ClickHouse), 2) + + await server.db.deletePerson(person) + + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).length === 0 ? ['deleted!'] : [] + ) + + const clickHousePersons = await server.db.fetchPersons(Database.ClickHouse) + const postgresPersons = await server.db.fetchPersons(Database.Postgres) + + expect(clickHousePersons.length).toEqual(0) + expect(postgresPersons.length).toEqual(0) + + const clickHouseDistinctIdValues = await server.db.fetchDistinctIdValues(person, Database.ClickHouse) + const postgresDistinctIdValues = await server.db.fetchDistinctIdValues(person, Database.Postgres) + expect(clickHouseDistinctIdValues.length).toEqual(0) + expect(postgresDistinctIdValues.length).toEqual(0) + }) + + test('addDistinctId & moveDistinctId', async () => { + const uuid = new UUIDT().toString() + const uuid2 = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + true, + uuid, + ['distinct1'] + ) + const anotherPerson = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + true, + uuid2, + ['another_distinct_id'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + const [postgresPerson] = await server.db.fetchPersons(Database.Postgres) + + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse), 1) + const clickHouseDistinctIdValues = await server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse) + const postgresDistinctIdValues = await server.db.fetchDistinctIdValues(postgresPerson, Database.Postgres) + + // check that all is in the right format + + expect(clickHouseDistinctIdValues).toEqual(['distinct1']) + expect(postgresDistinctIdValues).toEqual(['distinct1']) + + const clickHouseDistinctIds = await server.db.fetchDistinctIds(postgresPerson, Database.ClickHouse) + const postgresDistinctIds = await server.db.fetchDistinctIds(postgresPerson, Database.Postgres) + + expect(clickHouseDistinctIds).toEqual([ + { + id: expect.any(Number), + distinct_id: 'distinct1', + person_id: person.uuid, + team_id: team.id, + _timestamp: expect.any(String), + _offset: expect.any(Number), + }, + ]) + expect(postgresDistinctIds).toEqual([ + { + id: expect.any(Number), + distinct_id: 'distinct1', + person_id: person.id, + team_id: team.id, + }, + ]) + expect(clickHouseDistinctIds[0].id).toEqual(postgresDistinctIds[0].id) + + // add 'anotherOne' to person + + await server.db.addDistinctId(postgresPerson, 'anotherOne') + + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse), 2) + + const clickHouseDistinctIdValues2 = await server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse) + const postgresDistinctIdValues2 = await server.db.fetchDistinctIdValues(postgresPerson, Database.Postgres) + + expect(clickHouseDistinctIdValues2).toEqual(['distinct1', 'anotherOne']) + expect(postgresDistinctIdValues2).toEqual(['distinct1', 'anotherOne']) + + // check anotherPerson for their initial distinct id + + const clickHouseDistinctIdValuesOther = await server.db.fetchDistinctIdValues( + anotherPerson, + Database.ClickHouse + ) + const postgresDistinctIdValuesOther = await server.db.fetchDistinctIdValues(anotherPerson, Database.Postgres) + + expect(clickHouseDistinctIdValuesOther).toEqual(['another_distinct_id']) + expect(postgresDistinctIdValuesOther).toEqual(['another_distinct_id']) + + // move 'distinct1' from person to to anotherPerson + + await server.db.moveDistinctId(postgresPerson, postgresDistinctIds[0], anotherPerson) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(anotherPerson, Database.ClickHouse), 2) + + // it got added + + const clickHouseDistinctIdValuesMoved = await server.db.fetchDistinctIdValues( + anotherPerson, + Database.ClickHouse + ) + const postgresDistinctIdValuesMoved = await server.db.fetchDistinctIdValues(anotherPerson, Database.Postgres) + + expect(clickHouseDistinctIdValuesMoved).toEqual(['distinct1', 'another_distinct_id']) + expect(postgresDistinctIdValuesMoved).toEqual(['distinct1', 'another_distinct_id']) + + // it got removed + + const clickHouseDistinctIdValuesRemoved = await server.db.fetchDistinctIdValues( + postgresPerson, + Database.ClickHouse + ) + const postgresDistinctIdValuesRemoved = await server.db.fetchDistinctIdValues(postgresPerson, Database.Postgres) + + // The `distinct1` key is still there in clickhouse, yet ALSO there for the new person. + // Eventually this should be compacted away but it's not right now. + expect(clickHouseDistinctIdValuesRemoved).toEqual(['distinct1', 'anotherOne']) + expect(postgresDistinctIdValuesRemoved).toEqual(['anotherOne']) + }) +}) diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 7d1b535a..6c4ca51b 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -2,7 +2,14 @@ import { Kafka, logLevel } from 'kafkajs' import { PluginsServerConfig } from '../../src/types' import { delay, UUIDT } from '../../src/utils' import { defaultConfig, overrideWithEnv } from '../../src/config' -import { KAFKA_EVENTS_INGESTION_HANDOFF, KAFKA_SESSION_RECORDING_EVENTS } from '../../src/ingestion/topics' +import { + KAFKA_EVENTS, + KAFKA_EVENTS_INGESTION_HANDOFF, + KAFKA_EVENTS_WAL, + KAFKA_PERSON, + KAFKA_PERSON_UNIQUE_ID, + KAFKA_SESSION_RECORDING_EVENTS, +} from '../../src/ingestion/topics' /** Clear the kafka queue */ export async function resetKafka(extraServerConfig: Partial, delayMs = 2000) { @@ -19,25 +26,14 @@ export async function resetKafka(extraServerConfig: Partial }) const messages = [] - async function createTopic(topic: string) { - try { - const admin = kafka.admin() - await admin.connect() - await admin.createTopics({ - waitForLeaders: true, - topics: [{ topic }], - }) - await admin.disconnect() - } catch (e) { - if (!e?.error?.includes('Topic with this name already exists')) { - console.error(`Error creating kafka topic "${topic}". This might be fine!`) - console.error(e) - } - } - } - - await createTopic(KAFKA_EVENTS_INGESTION_HANDOFF) - await createTopic(KAFKA_SESSION_RECORDING_EVENTS) + await createTopics(kafka, [ + KAFKA_EVENTS, + KAFKA_EVENTS_INGESTION_HANDOFF, + KAFKA_EVENTS_WAL, + KAFKA_SESSION_RECORDING_EVENTS, + KAFKA_PERSON, + KAFKA_PERSON_UNIQUE_ID, + ]) const connected = await new Promise(async (resolve, reject) => { console.info('setting group join and crash listeners') @@ -76,3 +72,13 @@ export async function resetKafka(extraServerConfig: Partial return true } + +async function createTopics(kafka: Kafka, topics: string[]) { + const admin = kafka.admin() + await admin.connect() + await admin.createTopics({ + waitForLeaders: true, + topics: topics.map((topic) => ({ topic })), + }) + await admin.disconnect() +} diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 4c1aef36..35464c0e 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -2,7 +2,7 @@ import { makePluginObjects, commonOrganizationId, commonUserId, commonOrganizati import { defaultConfig } from '../../src/config' import { Pool } from 'pg' import { delay, UUIDT } from '../../src/utils' -import { PluginsServerConfig } from '../../src/types' +import { PluginsServer, PluginsServerConfig, Team } from '../../src/types' export async function resetTestDatabase( code: string, @@ -113,3 +113,11 @@ export async function createUserTeamAndOrganization( is_demo: false, }) } + +export async function getTeams(server: PluginsServer): Promise { + return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows +} + +export async function getFirstTeam(server: PluginsServer): Promise { + return (await getTeams(server))[0] +} diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 2c12142b..3288aea1 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -13,6 +13,8 @@ describe('process event (postgresql)', () => { const elementsHash = server!.db.createElementGroup(elements, 2) const elementGroup = await server!.db.fetchElements() + console.log(elementGroup) + expect(elementGroup[0].tag_name).toEqual('button') expect(elementGroup[1].tag_name).toEqual('div') expect(elementGroup.length).toEqual(2) diff --git a/tests/shared/process-event.ts b/tests/shared/process-event.ts index b9a4c058..249b67ab 100644 --- a/tests/shared/process-event.ts +++ b/tests/shared/process-event.ts @@ -12,7 +12,7 @@ import { ClickHouseEvent, SessionRecordingEvent, } from '../../src/types' -import { createUserTeamAndOrganization, resetTestDatabase } from '../helpers/sql' +import { createUserTeamAndOrganization, getFirstTeam, getTeams, resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' import { delay, UUIDT } from '../../src/utils' @@ -21,14 +21,6 @@ import { hashElements } from '../../src/ingestion/utils' jest.setTimeout(600000) // 600 sec timeout -async function getTeams(server: PluginsServer): Promise { - return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows -} - -async function getFirstTeam(server: PluginsServer): Promise { - return (await getTeams(server))[0] -} - export async function delayUntilEventIngested(fetchEvents: () => Promise, minCount = 1): Promise { for (let i = 0; i < 30; i++) { if ((await fetchEvents()).length >= minCount) { diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 27e1de0a..6cdef39c 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -8,6 +8,7 @@ import { UUID, UUIDT, sanitizeSqlIdentifier, + escapeClickHouseString, } from '../src/utils' import { randomBytes } from 'crypto' import { LogLevel } from '../src/types' @@ -306,3 +307,13 @@ describe('sanitizeSqlIdentifier', () => { expect(sanitizedIdentifier).toStrictEqual('some_fieldDROPTABLEactually_an_injection9') }) }) + +describe('escapeClickHouseString', () => { + it('escapes single quotes and slashes', () => { + const rawString = "insert'escape \\" + + const sanitizedString = escapeClickHouseString(rawString) + + expect(sanitizedString).toStrictEqual("insert\\'escape \\\\") + }) +}) From 947c8ba7a90bcabb4a8469e0832e3a0808ebc899 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 4 Feb 2021 13:49:14 +0100 Subject: [PATCH 214/218] Remove "_handoff" from Kafka topic (#127) * remove _HANDOFF from topic * add plugin_ to plugin server ingestion topic --- src/ingestion/topics.ts | 2 +- src/server.ts | 4 ++-- tests/clickhouse/e2e.test.ts | 4 ++-- tests/clickhouse/process-event.test.ts | 4 ++-- tests/helpers/kafka.ts | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ingestion/topics.ts b/src/ingestion/topics.ts index 05c6de36..a76767cb 100644 --- a/src/ingestion/topics.ts +++ b/src/ingestion/topics.ts @@ -3,4 +3,4 @@ export const KAFKA_PERSON = 'clickhouse_person' export const KAFKA_PERSON_UNIQUE_ID = 'clickhouse_person_unique_id' export const KAFKA_SESSION_RECORDING_EVENTS = 'clickhouse_session_recording_events' export const KAFKA_EVENTS_WAL = 'events_write_ahead_log' -export const KAFKA_EVENTS_INGESTION_HANDOFF = 'events_ingestion_handoff' +export const KAFKA_EVENTS_PLUGIN_INGESTION = 'events_plugin_ingestion' diff --git a/src/server.ts b/src/server.ts index 9442ce31..b05bec21 100644 --- a/src/server.ts +++ b/src/server.ts @@ -20,7 +20,7 @@ import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' import { DateTime } from 'luxon' -import { KAFKA_EVENTS_INGESTION_HANDOFF, KAFKA_EVENTS_WAL } from './ingestion/topics' +import { KAFKA_EVENTS_PLUGIN_INGESTION, KAFKA_EVENTS_WAL } from './ingestion/topics' export async function createServer( config: Partial = {}, @@ -86,7 +86,7 @@ export async function createServer( if (!serverConfig.KAFKA_CONSUMPTION_TOPIC) { // When ingesting events, listen to the "INGESTION_HANDOFF" topic, otherwise listen to the "WAL" and discard serverConfig.KAFKA_CONSUMPTION_TOPIC = serverConfig.PLUGIN_SERVER_INGESTION - ? KAFKA_EVENTS_INGESTION_HANDOFF + ? KAFKA_EVENTS_PLUGIN_INGESTION : KAFKA_EVENTS_WAL } diff --git a/tests/clickhouse/e2e.test.ts b/tests/clickhouse/e2e.test.ts index 51ac85f2..78996480 100644 --- a/tests/clickhouse/e2e.test.ts +++ b/tests/clickhouse/e2e.test.ts @@ -9,7 +9,7 @@ import { delay, UUIDT } from '../../src/utils' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { resetKafka } from '../helpers/kafka' import { delayUntilEventIngested } from '../shared/process-event' -import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' +import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../../src/ingestion/topics' jest.setTimeout(60000) // 60 sec timeout @@ -18,7 +18,7 @@ const extraServerConfig: Partial = { KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', WORKER_CONCURRENCY: 2, PLUGIN_SERVER_INGESTION: true, - KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_INGESTION_HANDOFF, + KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION, LOG_LEVEL: LogLevel.Log, } diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 41167ddf..a6a9b6a6 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -2,7 +2,7 @@ import { PluginsServerConfig, Event } from '../../src/types' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { resetKafka } from '../helpers/kafka' import { createProcessEventTests } from '../shared/process-event' -import { KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' +import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../../src/ingestion/topics' jest.setTimeout(180_000) // 3 minute timeout @@ -10,7 +10,7 @@ const extraServerConfig: Partial = { KAFKA_ENABLED: true, KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', PLUGIN_SERVER_INGESTION: true, - KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_INGESTION_HANDOFF, + KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION, } describe('process event (clickhouse)', () => { diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 6c4ca51b..031ffae8 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -4,7 +4,7 @@ import { delay, UUIDT } from '../../src/utils' import { defaultConfig, overrideWithEnv } from '../../src/config' import { KAFKA_EVENTS, - KAFKA_EVENTS_INGESTION_HANDOFF, + KAFKA_EVENTS_PLUGIN_INGESTION, KAFKA_EVENTS_WAL, KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID, @@ -28,7 +28,7 @@ export async function resetKafka(extraServerConfig: Partial await createTopics(kafka, [ KAFKA_EVENTS, - KAFKA_EVENTS_INGESTION_HANDOFF, + KAFKA_EVENTS_PLUGIN_INGESTION, KAFKA_EVENTS_WAL, KAFKA_SESSION_RECORDING_EVENTS, KAFKA_PERSON, @@ -50,7 +50,7 @@ export async function resetKafka(extraServerConfig: Partial await producer.connect() console.info('subscribing consumer') - await consumer.subscribe({ topic: KAFKA_EVENTS_INGESTION_HANDOFF }) + await consumer.subscribe({ topic: KAFKA_EVENTS_PLUGIN_INGESTION }) console.info('running consumer') await consumer.run({ eachMessage: async (payload) => { From 5a177d0494fab552b659b96f7722f0afeced73a9 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 4 Feb 2021 14:01:16 +0100 Subject: [PATCH 215/218] User merge test (#129) * merge test and postgres query simplification * postgres fix --- src/ingestion/process-event.ts | 15 +++----- tests/shared/process-event.ts | 67 +++++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 16 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index acc584e6..06657bdb 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -265,7 +265,7 @@ export class EventsProcessor { } } - private async mergePeople(mergeInto: Person, peopleToMerge: Person[]): Promise { + public async mergePeople(mergeInto: Person, peopleToMerge: Person[]): Promise { let firstSeen = mergeInto.created_at // merge the properties @@ -291,15 +291,10 @@ export class EventsProcessor { await this.db.moveDistinctId(otherPerson, personDistinctId, mergeInto) } - const otherCohortPeople: CohortPeople[] = ( - await this.db.postgresQuery('SELECT * FROM posthog_cohortpeople WHERE person_id = $1', [otherPerson.id]) - ).rows - for (const cohortPeople of otherCohortPeople) { - await this.db.postgresQuery('UPDATE posthog_cohortpeople SET person_id = $1 WHERE id = $2', [ - mergeInto.id, - cohortPeople.id, - ]) - } + await this.db.postgresQuery('UPDATE posthog_cohortpeople SET person_id = $1 WHERE person_id = $2', [ + mergeInto.id, + otherPerson.id, + ]) await this.db.deletePerson(otherPerson) } diff --git a/tests/shared/process-event.ts b/tests/shared/process-event.ts index 249b67ab..b5e8903e 100644 --- a/tests/shared/process-event.ts +++ b/tests/shared/process-event.ts @@ -1,16 +1,14 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' import { createServer } from '../../src/server' import { - LogLevel, - PluginsServer, - Team, + Database, Event, + LogLevel, Person, - Element, - PostgresSessionRecordingEvent, + PluginsServer, PluginsServerConfig, - ClickHouseEvent, SessionRecordingEvent, + Team, } from '../../src/types' import { createUserTeamAndOrganization, getFirstTeam, getTeams, resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' @@ -125,6 +123,63 @@ export const createProcessEventTests = ( createTests?.(returned) + test('merge people', async () => { + const p0 = await createPerson(server, team, ['person_0'], { $os: 'Microsoft' }) + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse), 1) + } + + await server.db.updatePerson(p0, { created_at: DateTime.fromISO('2020-01-01T00:00:00Z') }) + + const p1 = await createPerson(server, team, ['person_1'], { $os: 'Chrome' }) + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse), 2) + } + await server.db.updatePerson(p1, { created_at: DateTime.fromISO('2019-07-01T00:00:00Z') }) + + await processEvent( + 'person_1', + '', + '', + ({ + event: 'user signed up', + properties: {}, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + await createPerson(server, team, ['person_2'], { $os: 'Apple', $browser: 'MS Edge' }) + await createPerson(server, team, ['person_3'], { $os: 'PlayStation' }) + + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse), 4) + expect((await server.db.fetchPersons(Database.ClickHouse)).length).toEqual(4) + } + + expect((await server.db.fetchPersons()).length).toEqual(4) + const [person0, person1, person2, person3] = await server.db.fetchPersons() + + await eventsProcessor.mergePeople(person0, [person1, person2, person3]) + + if (database === 'clickhouse') { + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).length === 1 ? [1] : [] + ) + expect((await server.db.fetchPersons(Database.ClickHouse)).length).toEqual(1) + } + + expect((await server.db.fetchPersons()).length).toEqual(1) + + const [person] = await server.db.fetchPersons() + + expect(person.properties).toEqual({ $os: 'Microsoft', $browser: 'MS Edge' }) + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['person_0', 'person_1', 'person_2', 'person_3']) + expect(person.created_at.toISO()).toEqual(DateTime.fromISO('2019-07-01T00:00:00Z').setZone('UTC').toISO()) + }) + test('capture new person', async () => { await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [true, team.id]) team = await getFirstTeam(server) From 34de94e6169c72408ac7ed13367ff72d1153b677 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 4 Feb 2021 14:04:25 +0100 Subject: [PATCH 216/218] remove "_HANDOFF" --- src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index b05bec21..afe51cf7 100644 --- a/src/server.ts +++ b/src/server.ts @@ -84,7 +84,7 @@ export async function createServer( await clickhouse.query('SELECT 1') // test that the connection works if (!serverConfig.KAFKA_CONSUMPTION_TOPIC) { - // When ingesting events, listen to the "INGESTION_HANDOFF" topic, otherwise listen to the "WAL" and discard + // When ingesting events, listen to the "INGESTION" topic, otherwise listen to the "WAL" and discard serverConfig.KAFKA_CONSUMPTION_TOPIC = serverConfig.PLUGIN_SERVER_INGESTION ? KAFKA_EVENTS_PLUGIN_INGESTION : KAFKA_EVENTS_WAL From daa28757decb2403d2943d7342c3d8629cef7ef3 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 4 Feb 2021 14:10:00 +0100 Subject: [PATCH 217/218] small comment --- src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index afe51cf7..3a09dd7b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -101,7 +101,7 @@ export async function createServer( } // `node-postgres` will return dates as plain JS Date objects, which will use the local timezone. - // This converts all date fields to a proper luxon UTC DateTime + // This converts all date fields to a proper luxon UTC DateTime and then casts them to a string // Unfortunately this must be done on a global object before initializing the `Pool` pgTypes.setTypeParser(1083 /* types.TypeId.TIME */, (timeStr) => timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null From 849bbd482e9ab1b81e0934d1b030604021ffe836 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 4 Feb 2021 14:52:03 +0100 Subject: [PATCH 218/218] Switch ClickHouse driver (#128) * Use driver @posthog/clickhouse instead of clickhouse * Update CH querying * Fix clickhouseQuery usage * Don't quote 64-bit ints in JSON from CH --- package.json | 2 +- src/db.ts | 41 +++++++------- src/ingestion/process-event.ts | 2 +- src/server.ts | 19 ++++--- src/types.ts | 2 +- tests/helpers/clickhouse.ts | 24 +++++---- yarn.lock | 99 ++++------------------------------ 7 files changed, 59 insertions(+), 130 deletions(-) diff --git a/package.json b/package.json index f3d6d735..2e1aa6ad 100644 --- a/package.json +++ b/package.json @@ -37,10 +37,10 @@ "license": "MIT", "dependencies": { "@google-cloud/bigquery": "^5.5.0", + "@posthog/clickhouse": "^1.7.0", "@sentry/node": "^5.29.0", "@sentry/tracing": "^5.29.0", "adm-zip": "^0.4.16", - "clickhouse": "^2.2.1", "fastify": "^3.8.0", "hot-shots": "^8.2.1", "ioredis": "^4.19.2", diff --git a/src/db.ts b/src/db.ts index 4458298b..f80abfa7 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,9 +1,8 @@ import { Properties } from '@posthog/plugin-scaffold' -import { ClickHouse } from 'clickhouse' +import ClickHouse from '@posthog/clickhouse' import { Producer } from 'kafkajs' import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' -import { string } from 'yargs' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { chainToElements, hashElements, unparsePersonPartial } from './ingestion/utils' import { @@ -45,14 +44,17 @@ export class DB { queryTextOrConfig: string | QueryConfig, values?: I ): Promise> { - return this.postgres.query(queryTextOrConfig, values) + return await this.postgres.query(queryTextOrConfig, values) } - public async clickhouseQuery(query: string, reqParams?: Record): Promise> { + public async clickhouseQuery( + query: string, + options?: ClickHouse.QueryOptions + ): Promise>> { if (!this.clickhouse) { throw new Error('ClickHouse connection has not been provided to this DB instance!') } - return this.clickhouse.query(query, reqParams).toPromise() + return await this.clickhouse.querying(query, options) } // Person @@ -61,7 +63,7 @@ export class DB { public async fetchPersons(database: Database.ClickHouse): Promise public async fetchPersons(database: Database = Database.Postgres): Promise { if (database === Database.ClickHouse) { - return (await this.clickhouseQuery('SELECT * FROM person')) as ClickHousePerson[] + return (await this.clickhouseQuery('SELECT * FROM person')).data as ClickHousePerson[] } else if (database === Database.Postgres) { return ((await this.postgresQuery('SELECT * FROM posthog_person')).rows as RawPerson[]).map( (rawPerson: RawPerson) => @@ -186,11 +188,13 @@ export class DB { database: Database = Database.Postgres ): Promise { if (database === Database.ClickHouse) { - return (await this.clickhouseQuery( - `SELECT * FROM person_distinct_id WHERE person_id='${escapeClickHouseString( - person.uuid - )}' and team_id='${person.team_id}' ORDER BY id` - )) as ClickHousePersonDistinctId[] + return ( + await this.clickhouseQuery( + `SELECT * FROM person_distinct_id WHERE person_id='${escapeClickHouseString( + person.uuid + )}' and team_id='${person.team_id}' ORDER BY id` + ) + ).data as ClickHousePersonDistinctId[] } else if (database === Database.Postgres) { const result = await this.postgresQuery( 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', @@ -255,7 +259,7 @@ export class DB { public async fetchEvents(): Promise { if (this.kafkaProducer) { - const events = (await this.clickhouseQuery(`SELECT * FROM events`)) as ClickHouseEvent[] + const events = (await this.clickhouseQuery(`SELECT * FROM events`)).data as ClickHouseEvent[] return ( events?.map( (event) => @@ -278,9 +282,8 @@ export class DB { public async fetchSessionRecordingEvents(): Promise { if (this.kafkaProducer) { - const events = ((await this.clickhouseQuery( - `SELECT * FROM session_recording_events` - )) as SessionRecordingEvent[]).map((event) => { + const events = ((await this.clickhouseQuery(`SELECT * FROM session_recording_events`)) + .data as SessionRecordingEvent[]).map((event) => { return { ...event, snapshot_data: event.snapshot_data ? JSON.parse(event.snapshot_data) : null, @@ -297,9 +300,11 @@ export class DB { public async fetchElements(event?: Event): Promise { if (this.kafkaProducer) { - const events = (await this.clickhouseQuery( - `SELECT elements_chain FROM events WHERE uuid='${escapeClickHouseString((event as any).uuid)}'` - )) as ClickHouseEvent[] + const events = ( + await this.clickhouseQuery( + `SELECT elements_chain FROM events WHERE uuid='${escapeClickHouseString((event as any).uuid)}'` + ) + ).data as ClickHouseEvent[] const chain = events?.[0]?.elements_chain return chainToElements(chain) } else { diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 06657bdb..55b7db7c 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -16,7 +16,7 @@ import { Event as EventProto, IEvent } from '../idl/protos' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' import { elementsToString, sanitizeEventName } from './utils' -import { ClickHouse } from 'clickhouse' +import ClickHouse from '@posthog/clickhouse' import { DB } from '../db' import { status } from '../status' import * as Sentry from '@sentry/node' diff --git a/src/server.ts b/src/server.ts index 3a09dd7b..a17d2a18 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,7 +5,7 @@ import { Kafka, logLevel, Producer } from 'kafkajs' import { FastifyInstance } from 'fastify' import { PluginsServer, PluginsServerConfig, Queue } from './types' import { startQueue } from './worker/queue' -import { ClickHouse } from 'clickhouse' +import ClickHouse from '@posthog/clickhouse' import { startFastifyInstance, stopFastifyInstance } from './web/server' import { version } from '../package.json' import { PluginEvent } from '@posthog/plugin-scaffold' @@ -20,6 +20,7 @@ import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' import { DateTime } from 'luxon' +import * as fs from 'fs' import { KAFKA_EVENTS_PLUGIN_INGESTION, KAFKA_EVENTS_WAL } from './ingestion/topics' export async function createServer( @@ -71,17 +72,19 @@ export async function createServer( throw new Error('You must set KAFKA_HOSTS to process events from Kafka!') } clickhouse = new ClickHouse({ - url: `http${serverConfig.CLICKHOUSE_SECURE ? 's' : ''}://$${serverConfig.CLICKHOUSE_HOST}`, + host: serverConfig.CLICKHOUSE_HOST, port: serverConfig.CLICKHOUSE_SECURE ? 8443 : 8123, - basicAuth: { - username: serverConfig.CLICKHOUSE_USER, - password: serverConfig.CLICKHOUSE_PASSWORD, - }, - config: { + protocol: serverConfig.CLICKHOUSE_SECURE ? 'https:' : 'http:', + user: serverConfig.CLICKHOUSE_USER, + password: serverConfig.CLICKHOUSE_PASSWORD || undefined, + dataObjects: true, + queryOptions: { database: serverConfig.CLICKHOUSE_DATABASE, + output_format_json_quote_64bit_integers: false, }, + ca: serverConfig.CLICKHOUSE_CA ? fs.readFileSync(serverConfig.CLICKHOUSE_CA).toString() : undefined, }) - await clickhouse.query('SELECT 1') // test that the connection works + await clickhouse.querying('SELECT 1') // test that the connection works if (!serverConfig.KAFKA_CONSUMPTION_TOPIC) { // When ingesting events, listen to the "INGESTION" topic, otherwise listen to the "WAL" and discard diff --git a/src/types.ts b/src/types.ts index 4e70ef5c..d8bc7a53 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,7 +6,7 @@ import { VM } from 'vm2' import { DateTime } from 'luxon' import { StatsD } from 'hot-shots' import { EventsProcessor } from 'ingestion/process-event' -import { ClickHouse } from 'clickhouse' +import ClickHouse from '@posthog/clickhouse' import { DB } from './db' export enum LogLevel { diff --git a/tests/helpers/clickhouse.ts b/tests/helpers/clickhouse.ts index 9a006dd2..caa09f22 100644 --- a/tests/helpers/clickhouse.ts +++ b/tests/helpers/clickhouse.ts @@ -1,22 +1,24 @@ import { defaultConfig } from '../../src/config' -import { ClickHouse } from 'clickhouse' +import ClickHouse from '@posthog/clickhouse' import { PluginsServerConfig } from '../../src/types' export async function resetTestDatabaseClickhouse(extraServerConfig: Partial): Promise { const config = { ...defaultConfig, ...extraServerConfig } const clickhouse = new ClickHouse({ - url: `http://$${config.CLICKHOUSE_HOST}`, + host: config.CLICKHOUSE_HOST, port: 8123, - config: { + dataObjects: true, + queryOptions: { database: config.CLICKHOUSE_DATABASE, + output_format_json_quote_64bit_integers: false, }, }) - await clickhouse.query('TRUNCATE events').toPromise() - await clickhouse.query('TRUNCATE events_mv').toPromise() - await clickhouse.query('TRUNCATE person').toPromise() - await clickhouse.query('TRUNCATE person_distinct_id').toPromise() - await clickhouse.query('TRUNCATE person_mv').toPromise() - await clickhouse.query('TRUNCATE person_static_cohort').toPromise() - await clickhouse.query('TRUNCATE session_recording_events').toPromise() - await clickhouse.query('TRUNCATE session_recording_events_mv').toPromise() + await clickhouse.querying('TRUNCATE events') + await clickhouse.querying('TRUNCATE events_mv') + await clickhouse.querying('TRUNCATE person') + await clickhouse.querying('TRUNCATE person_distinct_id') + await clickhouse.querying('TRUNCATE person_mv') + await clickhouse.querying('TRUNCATE person_static_cohort') + await clickhouse.querying('TRUNCATE session_recording_events') + await clickhouse.querying('TRUNCATE session_recording_events_mv') } diff --git a/yarn.lock b/yarn.lock index a5470402..6e0c2e38 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1179,6 +1179,11 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" +"@posthog/clickhouse@^1.7.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@posthog/clickhouse/-/clickhouse-1.7.0.tgz#21fa1e8cfa0637b688f91964e0efeedbf4cf7a3c" + integrity sha512-B8hZ8Dh2EoJoDb7Gx38ylBQM92oON/X2IxXCb7BfYStk3m17nStcAyaCsc2zbvxC0fFfTMU8lFRiFSEJmijkyg== + "@posthog/plugin-scaffold@0.2.8": version "0.2.8" resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.8.tgz#14ee85afa9a91625bbae4e223ce03d215a22e8d7" @@ -1620,14 +1625,6 @@ "@typescript-eslint/types" "4.14.0" eslint-visitor-keys "^2.0.0" -JSONStream@1.3.4: - version "1.3.4" - resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.4.tgz#615bb2adb0cd34c8f4c447b5f6512fa1d8f16a2e" - integrity sha512-Y7vfi3I5oMOYIr+WxV8NZxDSwcbNgzdKYsTNInmycOq9bUYwGg9ryu57Wg5NLmCjqdFPNUmpMBo3kSJN9tCbXg== - dependencies: - jsonparse "^1.2.0" - through ">=2.2.7 <3" - abab@^2.0.3: version "2.0.5" resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" @@ -2289,20 +2286,6 @@ cli-truncate@^2.1.0: slice-ansi "^3.0.0" string-width "^4.2.0" -clickhouse@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/clickhouse/-/clickhouse-2.2.1.tgz#dc7c6c24a54a829ed2e7c4d5495852afe15ded1e" - integrity sha512-NcqKA1Uc353QKs1MepTSZZrv3gF4v5ScoobAoDxcC+37VKV1QusG/hkaZHTupJ28oppuRFpdWKSIoN54qkD2vg== - dependencies: - JSONStream "1.3.4" - lodash "4.17.19" - querystring "0.2.0" - request "2.88.0" - stream2asynciter "1.0.1" - through "2.3.8" - tsv "0.2.0" - uuid "3.4.0" - cliui@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-6.0.0.tgz#511d702c0c4e41ca156d7d0e96021f23e13225b1" @@ -3671,7 +3654,7 @@ har-schema@^2.0.0: resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= -har-validator@~5.1.0, har-validator@~5.1.3: +har-validator@~5.1.3: version "5.1.5" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== @@ -4733,11 +4716,6 @@ json5@^1.0.1: dependencies: minimist "^1.2.0" -jsonparse@^1.2.0: - version "1.3.1" - resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" - integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= - jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -4935,11 +4913,6 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg= -lodash@4.17.19: - version "4.17.19" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b" - integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ== - lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20: version "4.17.20" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" @@ -5884,7 +5857,7 @@ proxy-addr@^2.0.5: forwarded "~0.1.2" ipaddr.js "1.9.1" -psl@^1.1.24, psl@^1.1.28: +psl@^1.1.28: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== @@ -5897,11 +5870,6 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" -punycode@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" - integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= - punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" @@ -5917,11 +5885,6 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= - queue-microtask@^1.1.2: version "1.2.2" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.2.tgz#abf64491e6ecf0f38a6502403d4cda04f372dfd3" @@ -6177,32 +6140,6 @@ request-promise-native@^1.0.8: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request@2.88.0: - version "2.88.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" - integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - har-validator "~5.1.0" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - oauth-sign "~0.9.0" - performance-now "^2.1.0" - qs "~6.5.2" - safe-buffer "^5.1.2" - tough-cookie "~2.4.3" - tunnel-agent "^0.6.0" - uuid "^3.3.2" - request@^2.88.2: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" @@ -6712,11 +6649,6 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== -stream2asynciter@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/stream2asynciter/-/stream2asynciter-1.0.1.tgz#9f35c8fff36d89c02030b72e36702b61b6224d10" - integrity sha512-HseDrFlR3YhQEnZ4MAoQmW8poOkUeVfRiuv6t3+Wwb2ehNs4SSgzENcZoYEQdCgp/OFkt1IL2DpyvG9VctQXjg== - string-argv@0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da" @@ -6963,7 +6895,7 @@ throat@^5.0.0: resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b" integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA== -through@2.3.8, "through@>=2.2.7 <3", through@^2.3.8: +through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= @@ -7039,14 +6971,6 @@ tough-cookie@^3.0.1: psl "^1.1.28" punycode "^2.1.1" -tough-cookie@~2.4.3: - version "2.4.3" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" - integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ== - dependencies: - psl "^1.1.24" - punycode "^1.4.1" - tr46@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.0.2.tgz#03273586def1595ae08fedb38d7733cee91d2479" @@ -7142,11 +7066,6 @@ tsutils@^3.17.1: dependencies: tslib "^1.8.1" -tsv@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/tsv/-/tsv-0.2.0.tgz#92869a3cb5f50332f3dc90fca82be667db6f72d6" - integrity sha1-koaaPLX1AzLz3JD8qCvmZ9tvctY= - tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" @@ -7291,7 +7210,7 @@ util-deprecate@^1.0.1, util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= -uuid@3.4.0, uuid@^3.3.2: +uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==