diff --git a/.eslintrc.js b/.eslintrc.js index c5c9d271..4ee0805d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,7 +1,7 @@ module.exports = { parser: '@typescript-eslint/parser', plugins: ['@typescript-eslint'], - extends: ['plugin:@typescript-eslint/recommended', 'prettier/@typescript-eslint'], + extends: ['plugin:@typescript-eslint/recommended', 'prettier', 'prettier/@typescript-eslint', 'prettier/react'], ignorePatterns: ['bin', 'dist', 'node_modules'], rules: { '@typescript-eslint/no-unused-vars': 'off', diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70bd80f1..d0030629 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,8 +73,15 @@ jobs: with: node-version: 14 + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('posthog/requirements.txt') }} + - name: Install requirements.txt dependencies with pip - run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt + run: | + pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r posthog/requirements.txt - name: Set up databases env: @@ -93,6 +100,97 @@ jobs: REDIS_URL: 'redis://localhost' run: cd plugin-server && yarn test:postgres + tests-clickhouse: + name: Tests / ClickHouse + Kafka + runs-on: ubuntu-20.04 + + services: + postgres: + image: postgres:12 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: test_posthog + ports: ['5432:5432'] + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis + ports: + - '6379:6379' + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + env: + REDIS_URL: 'redis://localhost' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_DATABASE: 'posthog_test' + KAFKA_ENABLED: 'true' + KAFKA_HOSTS: 'kafka:9092' + + steps: + - name: Check out Django server for database setup + uses: actions/checkout@v2 + with: + repository: 'PostHog/posthog' + path: 'posthog/' + + - name: Check out plugin server + uses: actions/checkout@v2 + with: + path: 'plugin-server' + + - name: Fix Kafka Hostname + run: | + sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts' + ping -c 1 kafka + ping -c 1 zookeeper + + - name: Start Kafka, Clickhouse, Zookeeper + run: | + cd posthog/ee + docker-compose -f docker-compose.ch.yml up -d zookeeper kafka clickhouse + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Set up Node 14 + uses: actions/setup-node@v2 + with: + node-version: 14 + + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('posthog/requirements.txt') }} + + - name: Install requirements.txt dependencies with pip + run: | + pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r posthog/requirements.txt + + - name: Set up databases + env: + SECRET_KEY: 'abcdef' # unsafe - for testing only + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/posthog' + PRIMARY_DB: 'clickhouse' + TEST: 'true' + run: python posthog/manage.py setup_test_environment + + - name: Install package.json dependencies with Yarn + run: cd plugin-server && yarn + + - name: Test with Jest + env: + # Below DB name has `test_` prepended, as that's how Django (ran above) creates the test DB + DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/test_posthog' + REDIS_URL: 'redis://localhost' + run: cd plugin-server && yarn test:clickhouse + benchmarks-postgres: name: Benchmarks / Postgres + Redis runs-on: ubuntu-20.04 @@ -141,8 +239,15 @@ jobs: with: node-version: 14 + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('posthog/requirements.txt') }} + - name: Install requirements.txt dependencies with pip - run: python -m pip install --upgrade pip && python -m pip install -r posthog/requirements.txt + run: | + pip install --upgrade pip + pip install --upgrade --upgrade-strategy eager -r posthog/requirements.txt - name: Set up databases env: diff --git a/.gitignore b/.gitignore index 8aa1a5b5..3f6480bc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ yarn-error.log dist/ .yalc/ yalc.lock +src/idl/protos.* diff --git a/Dockerfile b/Dockerfile index e754e9a9..e33f6bdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,10 @@ FROM node:14 WORKDIR /code COPY package.json yarn.lock .eslintrc.js .prettierrc ./ +COPY src/idl/ src/idl/ RUN yarn install --frozen-lockfile COPY ./ ./ -RUN yarn compile +RUN yarn compile:typescript CMD [ "node", "dist/src/index.js" ] diff --git a/README.md b/README.md index 0ccc5d1f..6d1e254b 100644 --- a/README.md +++ b/README.md @@ -27,30 +27,38 @@ Let's get you developing the plugin server in no time: There's a multitude of settings you can use to control the plugin server. Use them as environment variables. -| Name | Description | Default value | -| ----------------------------- | ---------------------------------------------------------- | ------------------------------------- | -| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | -| REDIS_URL | Redis store URL | `'redis://localhost'` | -| BASE_DIR | base path for resolving local plugins | `'.'` | -| WORKER_CONCURRENCY | number of concurrent worker threads | `0` – all cores | -| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | -| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | -| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | -| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | -| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | -| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | -| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | -| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | -| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | -| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | -| DISABLE_WEB | whether to disable web server | `true` | -| WEB_PORT | port for web server to listen on | `3008` | -| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | -| LOG_LEVEL | minimum log level | `LogLevel.Info` | -| SENTRY_DSN | Sentry ingestion URL | `null` | -| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | -| STATSD_PORT | StatsD port | `8125` | -| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | +| Name | Description | Default value | +| ----------------------------- | ------------------------------------------------------------------- | ------------------------------------- | +| DATABASE_URL | Postgres database URL | `'postgres://localhost:5432/posthog'` | +| REDIS_URL | Redis store URL | `'redis://localhost'` | +| BASE_DIR | base path for resolving local plugins | `'.'` | +| WORKER_CONCURRENCY | number of concurrent worker threads | `0` – all cores | +| TASKS_PER_WORKER | number of parallel tasks per worker thread | `10` | +| SCHEDULE_LOCK_TTL | How many seconds to hold the lock for the schedule | `60` | +| CELERY_DEFAULT_QUEUE | Celery outgoing queue | `'celery'` | +| PLUGINS_CELERY_QUEUE | Celery incoming queue | `'posthog-plugins'` | +| PLUGINS_RELOAD_PUBSUB_CHANNEL | Redis channel for reload events | `'reload-plugins'` | +| PLUGIN_SERVER_INGESTION | Whether the plugin server should put events right into the database | `false` | +| CLICKHOUSE_HOST | ClickHouse host | `'localhost'` | +| CLICKHOUSE_DATABASE | ClickHouse database | `'default'` | +| CLICKHOUSE_USER | ClickHouse username | `'default'` | +| CLICKHOUSE_PASSWORD | ClickHouse password | `null` | +| CLICKHOUSE_CA | ClickHouse CA certs | `null` | +| CLICKHOUSE_SECURE | Secure ClickHouse connection | `false` | +| KAFKA_ENABLED | use Kafka instead of Celery to ingest events | `false` | +| KAFKA_HOSTS | comma-delimited Kafka hosts | `null` | +| KAFKA_CONSUMPTION_TOPIC | Kafka consumption topic override | `null` (automatic) | +| KAFKA_CLIENT_CERT_B64 | Kafka certificate in Base64 | `null` | +| KAFKA_CLIENT_CERT_KEY_B64 | Kafka certificate key in Base64 | `null` | +| KAFKA_TRUSTED_CERT_B64 | Kafka trusted CA in Base64 | `null` | +| DISABLE_WEB | whether to disable web server | `true` | +| WEB_PORT | port for web server to listen on | `3008` | +| WEB_HOSTNAME | hostname for web server to listen on | `'0.0.0.0'` | +| LOG_LEVEL | minimum log level | `LogLevel.Info` | +| SENTRY_DSN | Sentry ingestion URL | `null` | +| STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | +| STATSD_PORT | StatsD port | `8125` | +| STATSD_PREFIX | StatsD prefix | `'plugin-server.'` | ## Questions? diff --git a/jest.config.js b/jest.config.js index c8474d90..c4967502 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,6 +3,6 @@ module.exports = { testEnvironment: 'node', clearMocks: true, coverageProvider: 'v8', - setupFilesAfterEnv: ['./jest.setup.kafka-mock.js', './jest.setup.fetch-mock.js'], + setupFilesAfterEnv: ['./jest.setup.fetch-mock.js'], testMatch: ['/tests/**/*.test.ts', '/benchmarks/**/*.benchmark.ts'], } diff --git a/jest.setup.kafka-mock.js b/jest.setup.kafka-mock.js deleted file mode 100644 index 67300f89..00000000 --- a/jest.setup.kafka-mock.js +++ /dev/null @@ -1 +0,0 @@ -jest.mock('kafkajs') diff --git a/package.json b/package.json index ee6ec5ae..2e1aa6ad 100644 --- a/package.json +++ b/package.json @@ -5,21 +5,29 @@ "types": "dist/src/index.d.ts", "main": "dist/src/index.js", "scripts": { - "test": "jest --runInBand tests/*.test.ts", - "test:postgres": "yarn test --testPathIgnorePatterns '.*/clickhouse'", - "test:clickhouse": "yarn test --testPathIgnorePatterns '.*/postgres'", + "test": "jest --runInBand tests/**/*.test.ts", + "test:postgres": "jest --runInBand tests/postgres/*.test.ts tests/*.test.ts", + "test:clickhouse": "jest --runInBand tests/clickhouse/*.test.ts", "benchmark": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/", "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", "start:dev": "ts-node-dev --exit-child src/index.ts --base-dir ../posthog", + "start:dev:ee": "KAFKA_ENABLED=true KAFKA_HOSTS=localhost:9092 yarn start:dev", "build": "yarn clean && yarn compile", "clean": "rimraf dist/*", - "compile": "tsc -p .", + "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && eslint --fix . && prettier --write .", + "compile:typescript": "tsc", + "compile": "yarn compile:protobuf && yarn compile:typescript", "lint": "eslint .", "lint:fix": "eslint --fix .", "prettier": "prettier --write .", "prettier:check": "prettier --check .", - "prepublishOnly": "yarn build" + "prepare": "yarn compile:protobuf", + "prepublishOnly": "yarn build", + "setup:dev:clickhouse": "cd ../posthog && export DEBUG=1 PRIMARY_DB=clickhouse && source env/bin/activate && python manage.py migrate_clickhouse", + "setup:test:ee": "yarn setup:test:postgres && yarn setup:test:clickhouse", + "setup:test:postgres": "cd ../posthog && (dropdb test_posthog || echo 'no db to drop') && createdb test_posthog && source env/bin/activate && DATABASE_URL=postgres://localhost:5432/test_posthog DEBUG=1 python manage.py migrate", + "setup:test:clickhouse": "cd ../posthog && export TEST=1 PRIMARY_DB=clickhouse CLICKHOUSE_DATABASE=posthog_test && source env/bin/activate && python manage.py migrate_clickhouse" }, "bin": { "posthog-plugin-server": "bin/posthog-plugin-server" @@ -29,9 +37,9 @@ "license": "MIT", "dependencies": { "@google-cloud/bigquery": "^5.5.0", + "@posthog/clickhouse": "^1.7.0", "@sentry/node": "^5.29.0", "@sentry/tracing": "^5.29.0", - "@types/luxon": "^1.25.0", "adm-zip": "^0.4.16", "fastify": "^3.8.0", "hot-shots": "^8.2.1", @@ -42,6 +50,8 @@ "node-schedule": "^1.3.2", "pg": "^8.4.2", "piscina": "^2.1.0", + "posthog-js-lite": "^0.0.5", + "protobufjs": "^6.10.2", "redlock": "^4.2.0", "tar-stream": "^2.1.4", "uuid": "^8.3.1", @@ -53,10 +63,11 @@ "@babel/core": "^7.0.0", "@babel/preset-env": "^7.0.0", "@babel/preset-typescript": "^7.8.3", - "@posthog/plugin-scaffold": "0.2.6", + "@posthog/plugin-scaffold": "0.2.8", "@types/adm-zip": "^0.4.33", "@types/ioredis": "^4.17.7", "@types/jest": "^26.0.15", + "@types/luxon": "^1.25.0", "@types/node": "^14.14.6", "@types/node-fetch": "^2.5.7", "@types/node-schedule": "^1.3.1", @@ -95,6 +106,6 @@ }, "lint-staged": { "*.{js,ts}": "eslint --fix", - "*.{js,ts,css,scss,json,yml}": "prettier --write" + "*.{js,ts,css,scss,json,yml,md}": "prettier --write" } } diff --git a/src/celery/worker.ts b/src/celery/worker.ts index 19d96f7f..107dad04 100644 --- a/src/celery/worker.ts +++ b/src/celery/worker.ts @@ -171,14 +171,14 @@ export class Worker extends Base implements Queue { throw new Error(`Missing process handler for task ${taskName}`) } - console.info( + console.debug( `celery.node Received task: ${taskName}[${taskId}], args: ${args}, kwargs: ${JSON.stringify(kwargs)}` ) const timeStart = process.hrtime() const taskPromise = handler(...args, kwargs).then((result) => { const diff = process.hrtime(timeStart) - console.info( + console.debug( `celery.node Task ${taskName}[${taskId}] succeeded in ${diff[0] + diff[1] / 1e9}s: ${result}` ) this.activeTasks.delete(taskPromise) diff --git a/src/config.ts b/src/config.ts index 62a48e5b..f0888db2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,12 +8,20 @@ export function getDefaultConfig(): PluginsServerConfig { return { CELERY_DEFAULT_QUEUE: 'celery', - DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/posthog_test' : 'postgres://localhost:5432/posthog', + DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/test_posthog' : 'postgres://localhost:5432/posthog', + CLICKHOUSE_HOST: 'localhost', + CLICKHOUSE_DATABASE: isTestEnv ? 'posthog_test' : 'default', + CLICKHOUSE_USER: 'default', + CLICKHOUSE_PASSWORD: null, + CLICKHOUSE_CA: null, + CLICKHOUSE_SECURE: false, KAFKA_ENABLED: false, KAFKA_HOSTS: null, KAFKA_CLIENT_CERT_B64: null, KAFKA_CLIENT_CERT_KEY_B64: null, KAFKA_TRUSTED_CERT_B64: null, + KAFKA_CONSUMPTION_TOPIC: null, + PLUGIN_SERVER_INGESTION: false, PLUGINS_CELERY_QUEUE: 'posthog-plugins', REDIS_URL: 'redis://127.0.0.1', BASE_DIR: '.', @@ -34,9 +42,16 @@ export function getDefaultConfig(): PluginsServerConfig { export function getConfigHelp(): Record { return { + PLUGIN_SERVER_INGESTION: 'Ingest events via plugin-server', CELERY_DEFAULT_QUEUE: 'Celery outgoing queue', PLUGINS_CELERY_QUEUE: 'Celery incoming queue', DATABASE_URL: 'Postgres database URL', + CLICKHOUSE_HOST: 'ClickHouse host', + CLICKHOUSE_DATABASE: 'ClickHouse database', + CLICKHOUSE_USER: 'ClickHouse username', + CLICKHOUSE_PASSWORD: 'ClickHouse password', + CLICKHOUSE_CA: 'ClickHouse CA certs', + CLICKHOUSE_SECURE: 'Secure ClickHouse connection', REDIS_URL: 'Redis store URL', BASE_DIR: 'base path for resolving local plugins', PLUGINS_RELOAD_PUBSUB_CHANNEL: 'Redis channel for reload events', @@ -48,6 +63,7 @@ export function getConfigHelp(): Record { LOG_LEVEL: 'minimum log level', KAFKA_ENABLED: 'use Kafka instead of Celery to ingest events', KAFKA_HOSTS: 'comma-delimited Kafka hosts', + KAFKA_CONSUMPTION_TOPIC: 'Kafka consumption topic override', KAFKA_CLIENT_CERT_B64: 'Kafka certificate in Base64', KAFKA_CLIENT_CERT_KEY_B64: 'Kafka certificate key in Base64', KAFKA_TRUSTED_CERT_B64: 'Kafka trusted CA in Base64', diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 00000000..f80abfa7 --- /dev/null +++ b/src/db.ts @@ -0,0 +1,353 @@ +import { Properties } from '@posthog/plugin-scaffold' +import ClickHouse from '@posthog/clickhouse' +import { Producer } from 'kafkajs' +import { DateTime } from 'luxon' +import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' +import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' +import { chainToElements, hashElements, unparsePersonPartial } from './ingestion/utils' +import { + ClickHouseEvent, + ClickHousePerson, + ClickHousePersonDistinctId, + Database, + Element, + ElementGroup, + Event, + Person, + PersonDistinctId, + PostgresSessionRecordingEvent, + RawOrganization, + RawPerson, + SessionRecordingEvent, + TimestampFormat, +} from './types' +import { castTimestampOrNow, clickHouseTimestampToISO, escapeClickHouseString, sanitizeSqlIdentifier } from './utils' + +/** The recommended way of accessing the database. */ +export class DB { + /** Postgres connection pool for primary database access. */ + postgres: Pool + /** Kafka producer used for syncing Postgres and ClickHouse person data. */ + kafkaProducer?: Producer + /** ClickHouse used for syncing Postgres and ClickHouse person data. */ + clickhouse?: ClickHouse + + constructor(postgres: Pool, kafkaProducer?: Producer, clickhouse?: ClickHouse) { + this.postgres = postgres + this.kafkaProducer = kafkaProducer + this.clickhouse = clickhouse + } + + // Direct queries + + public async postgresQuery( + queryTextOrConfig: string | QueryConfig, + values?: I + ): Promise> { + return await this.postgres.query(queryTextOrConfig, values) + } + + public async clickhouseQuery( + query: string, + options?: ClickHouse.QueryOptions + ): Promise>> { + if (!this.clickhouse) { + throw new Error('ClickHouse connection has not been provided to this DB instance!') + } + return await this.clickhouse.querying(query, options) + } + + // Person + + public async fetchPersons(database?: Database.Postgres): Promise + public async fetchPersons(database: Database.ClickHouse): Promise + public async fetchPersons(database: Database = Database.Postgres): Promise { + if (database === Database.ClickHouse) { + return (await this.clickhouseQuery('SELECT * FROM person')).data as ClickHousePerson[] + } else if (database === Database.Postgres) { + return ((await this.postgresQuery('SELECT * FROM posthog_person')).rows as RawPerson[]).map( + (rawPerson: RawPerson) => + ({ + ...rawPerson, + created_at: DateTime.fromISO(rawPerson.created_at).toUTC(), + } as Person) + ) + } else { + throw new Error(`Can't fetch persons for database: ${database}`) + } + } + + public async fetchPerson(teamId: number, distinctId: string): Promise { + const selectResult = await this.postgresQuery( + `SELECT + posthog_person.id, posthog_person.created_at, posthog_person.team_id, posthog_person.properties, + posthog_person.is_user_id, posthog_person.is_identified, posthog_person.uuid, + posthog_persondistinctid.team_id AS persondistinctid__team_id, + posthog_persondistinctid.distinct_id AS persondistinctid__distinct_id + FROM posthog_person + JOIN posthog_persondistinctid ON (posthog_persondistinctid.person_id = posthog_person.id) + WHERE + posthog_person.team_id = $1 + AND posthog_persondistinctid.team_id = $1 + AND posthog_persondistinctid.distinct_id = $2`, + [teamId, distinctId] + ) + if (selectResult.rows.length > 0) { + const rawPerson: RawPerson = selectResult.rows[0] + return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at).toUTC() } + } + } + + public async createPerson( + createdAt: DateTime, + properties: Properties, + teamId: number, + isUserId: number | null, + isIdentified: boolean, + uuid: string, + distinctIds?: string[] + ): Promise { + const insertResult = await this.postgresQuery( + 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', + [createdAt.toISO(), JSON.stringify(properties), teamId, isUserId, isIdentified, uuid] + ) + const personCreated = insertResult.rows[0] as RawPerson + const person = { ...personCreated, created_at: DateTime.fromISO(personCreated.created_at).toUTC() } as Person + if (this.kafkaProducer) { + const data = { + created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouse), + properties: JSON.stringify(properties), + team_id: teamId, + is_identified: isIdentified, + id: uuid, + } + await this.kafkaProducer.send({ + topic: KAFKA_PERSON, + messages: [{ value: Buffer.from(JSON.stringify(data)) }], + }) + } + + for (const distinctId of distinctIds || []) { + await this.addDistinctId(person, distinctId) + } + return person + } + + public async updatePerson(person: Person, update: Partial): Promise { + const updatedPerson: Person = { ...person, ...update } + const values = [...Object.values(unparsePersonPartial(update)), person.id] + await this.postgresQuery( + `UPDATE posthog_person SET ${Object.keys(update).map( + (field, index) => `"${sanitizeSqlIdentifier(field)}" = $${index + 1}` + )} WHERE id = $${Object.values(update).length + 1}`, + values + ) + if (this.clickhouse) { + const { is_user_id, id, uuid, ...validUpdates } = update + const updateString = Object.entries(validUpdates) + .map(([key, value]) => { + let clickhouseValue: string + if (typeof value === 'string') { + clickhouseValue = value + } else if (typeof value === 'boolean') { + clickhouseValue = value ? '1' : '0' + } else if (DateTime.isDateTime(value)) { + clickhouseValue = castTimestampOrNow(value, TimestampFormat.ClickHouse) + } else { + clickhouseValue = JSON.stringify(value) + } + return `${sanitizeSqlIdentifier(key)} = '${escapeClickHouseString(clickhouseValue)}'` + }) + .join(', ') + if (updateString.length > 0) { + await this.clickhouseQuery( + `ALTER TABLE person UPDATE ${updateString} WHERE id = '${escapeClickHouseString(person.uuid)}'` + ) + } + } + return updatedPerson + } + + public async deletePerson(person: Person): Promise { + await this.postgresQuery('DELETE FROM posthog_persondistinctid WHERE person_id = $1', [person.id]) + await this.postgresQuery('DELETE FROM posthog_person WHERE id = $1', [person.id]) + if (this.clickhouse) { + await this.clickhouseQuery(`ALTER TABLE person DELETE WHERE id = '${escapeClickHouseString(person.uuid)}'`) + await this.clickhouseQuery( + `ALTER TABLE person_distinct_id DELETE WHERE person_id = '${escapeClickHouseString(person.uuid)}'` + ) + } + } + + // PersonDistinctId + + public async fetchDistinctIds(person: Person, database?: Database.Postgres): Promise + public async fetchDistinctIds(person: Person, database: Database.ClickHouse): Promise + public async fetchDistinctIds( + person: Person, + database: Database = Database.Postgres + ): Promise { + if (database === Database.ClickHouse) { + return ( + await this.clickhouseQuery( + `SELECT * FROM person_distinct_id WHERE person_id='${escapeClickHouseString( + person.uuid + )}' and team_id='${person.team_id}' ORDER BY id` + ) + ).data as ClickHousePersonDistinctId[] + } else if (database === Database.Postgres) { + const result = await this.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', + [person.id, person.team_id] + ) + return result.rows as PersonDistinctId[] + } else { + throw new Error(`Can't fetch persons for database: ${database}`) + } + } + + public async fetchDistinctIdValues(person: Person, database: Database = Database.Postgres): Promise { + const personDistinctIds = await this.fetchDistinctIds(person, database as any) + return personDistinctIds.map((pdi) => pdi.distinct_id) + } + + public async addDistinctId(person: Person, distinctId: string): Promise { + const insertResult = await this.postgresQuery( + 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', + [distinctId, person.id, person.team_id] + ) + const personDistinctIdCreated = insertResult.rows[0] as PersonDistinctId + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_PERSON_UNIQUE_ID, + messages: [ + { value: Buffer.from(JSON.stringify({ ...personDistinctIdCreated, person_id: person.uuid })) }, + ], + }) + } + } + + public async moveDistinctId( + person: Person, + personDistinctId: PersonDistinctId, + moveToPerson: Person + ): Promise { + await this.postgresQuery(`UPDATE posthog_persondistinctid SET person_id = $1 WHERE id = $2`, [ + moveToPerson.id, + personDistinctId.id, + ]) + if (this.kafkaProducer) { + const clickhouseModel: ClickHousePersonDistinctId = { ...personDistinctId, person_id: moveToPerson.uuid } + await this.kafkaProducer.send({ + topic: KAFKA_PERSON_UNIQUE_ID, + messages: [{ value: Buffer.from(JSON.stringify(clickhouseModel)) }], + }) + } + } + + // Organization + + public async fetchOrganization(organizationId: string): Promise { + const selectResult = await this.postgresQuery(`SELECT * FROM posthog_organization WHERE id $1`, [ + organizationId, + ]) + const rawOrganization: RawOrganization = selectResult.rows[0] + return rawOrganization + } + + // Event + + public async fetchEvents(): Promise { + if (this.kafkaProducer) { + const events = (await this.clickhouseQuery(`SELECT * FROM events`)).data as ClickHouseEvent[] + return ( + events?.map( + (event) => + ({ + ...event, + ...(typeof event['properties'] === 'string' + ? { properties: JSON.parse(event.properties) } + : {}), + timestamp: clickHouseTimestampToISO(event.timestamp), + } as ClickHouseEvent) + ) || [] + ) + } else { + const result = await this.postgresQuery('SELECT * FROM posthog_event') + return result.rows as Event[] + } + } + + // SessionRecordingEvent + + public async fetchSessionRecordingEvents(): Promise { + if (this.kafkaProducer) { + const events = ((await this.clickhouseQuery(`SELECT * FROM session_recording_events`)) + .data as SessionRecordingEvent[]).map((event) => { + return { + ...event, + snapshot_data: event.snapshot_data ? JSON.parse(event.snapshot_data) : null, + } + }) + return events + } else { + const result = await this.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') + return result.rows as PostgresSessionRecordingEvent[] + } + } + + // Element + + public async fetchElements(event?: Event): Promise { + if (this.kafkaProducer) { + const events = ( + await this.clickhouseQuery( + `SELECT elements_chain FROM events WHERE uuid='${escapeClickHouseString((event as any).uuid)}'` + ) + ).data as ClickHouseEvent[] + const chain = events?.[0]?.elements_chain + return chainToElements(chain) + } else { + return (await this.postgresQuery('SELECT * FROM posthog_element')).rows + } + } + + public async createElementGroup(elements: Element[], teamId: number): Promise { + const cleanedElements = elements.map((element, index) => ({ ...element, order: index })) + const hash = hashElements(cleanedElements) + + try { + const insertResult = await this.postgresQuery( + 'INSERT INTO posthog_elementgroup (hash, team_id) VALUES ($1, $2) RETURNING *', + [hash, teamId] + ) + const elementGroup = insertResult.rows[0] as ElementGroup + for (const element of cleanedElements) { + await this.postgresQuery( + 'INSERT INTO posthog_element (text, tag_name, href, attr_id, nth_child, nth_of_type, attributes, "order", event_id, attr_class, group_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)', + [ + element.text, + element.tag_name, + element.href, + element.attr_id, + element.nth_child, + element.nth_of_type, + element.attributes || '{}', + element.order, + element.event_id, + element.attr_class, + elementGroup.id, + ] + ) + } + } catch (error) { + // Throw further if not postgres error nr "23505" == "unique_violation" + // https://www.postgresql.org/docs/12/errcodes-appendix.html + if (error.code !== '23505') { + throw error + } + } + + return hash + } +} diff --git a/src/extensions/posthog.ts b/src/extensions/posthog.ts index a0ba28fa..863d662d 100644 --- a/src/extensions/posthog.ts +++ b/src/extensions/posthog.ts @@ -1,10 +1,19 @@ -import { KAFKA_EVENTS_WAL } from '../ingestion/topics' +import { Properties } from '@posthog/plugin-scaffold' import { DateTime } from 'luxon' import { PluginsServer, PluginConfig, RawEventMessage } from 'types' import { version } from '../../package.json' import Client from '../celery/client' import { UUIDT } from '../utils' +interface InternalData { + distinct_id: string + event: string + timestamp: string + properties: Properties + team_id: number + uuid: string +} + export interface DummyPostHog { capture(event: string, properties?: Record): void } @@ -12,68 +21,61 @@ export interface DummyPostHog { export function createPosthog(server: PluginsServer, pluginConfig: PluginConfig): DummyPostHog { const distinctId = pluginConfig.plugin?.name || `plugin-id-${pluginConfig.plugin_id}` - const client = server.KAFKA_ENABLED ? null : new Client(server.redis, server.PLUGINS_CELERY_QUEUE) // Redis - const producer = server.KAFKA_ENABLED ? server.kafka!.producer() : null // Kafka - producer?.connect() + let sendEvent: (data: InternalData) => Promise - function sendEventRedis(event: string, properties: Record, timestamp: string) { - const data = { - distinct_id: distinctId, - event, - timestamp, - properties: { - $lib: 'posthog-plugin-server', - $lib_version: version, - ...properties, - }, + if (server.KAFKA_ENABLED) { + // Sending event to our Kafka>ClickHouse pipeline + sendEvent = async (data) => { + if (!server.kafkaProducer) { + throw new Error('kafkaProducer not configured!') + } + server.kafkaProducer.send({ + topic: server.KAFKA_CONSUMPTION_TOPIC!, + messages: [ + { + key: data.uuid, + value: JSON.stringify({ + distinct_id: data.distinct_id, + ip: '', + site_url: '', + data: JSON.stringify(data), + team_id: pluginConfig.team_id, + now: data.timestamp, + sent_at: data.timestamp, + uuid: data.uuid, + } as RawEventMessage), + }, + ], + }) } - - client!.sendTask( - 'posthog.tasks.process_event.process_event_with_plugins', - [distinctId, null, null, data, pluginConfig.team_id, timestamp, timestamp], - {} - ) - } - - function sendEventKafka(event: string, properties: Record, timestamp: string) { - const uuid = new UUIDT().toString() - const data = { - distinct_id: distinctId, - event, - timestamp, - properties: { - $lib: 'posthog-plugin-server', - $lib_version: version, - ...properties, - }, + } else { + // Sending event to our Redis>Postgres pipeline + const client = new Client(server.redis, server.PLUGINS_CELERY_QUEUE) + sendEvent = async (data) => { + client.sendTask( + 'posthog.tasks.process_event.process_event_with_plugins', + [data.distinct_id, null, null, data, pluginConfig.team_id, data.timestamp, data.timestamp], + {} + ) } - - producer!.send({ - topic: KAFKA_EVENTS_WAL, - messages: [ - { - key: uuid, - value: JSON.stringify({ - distinct_id: distinctId, - ip: '', - site_url: '', - data: JSON.stringify(data), - team_id: pluginConfig.team_id, - now: timestamp, - sent_at: timestamp, - uuid, - } as RawEventMessage), - }, - ], - }) } - const sendEvent = server.KAFKA_ENABLED ? sendEventKafka : sendEventRedis - return { capture(event, properties = {}) { - const { timestamp, ...otherProperties } = properties - sendEvent(event, otherProperties, timestamp || DateTime.utc().toISO()) + const { timestamp = DateTime.utc().toISO(), ...otherProperties } = properties + const data: InternalData = { + distinct_id: distinctId, + event, + timestamp, + properties: { + $lib: 'posthog-plugin-server', + $lib_version: version, + ...otherProperties, + }, + team_id: pluginConfig.team_id, + uuid: new UUIDT().toString(), + } + sendEvent(data) }, } } diff --git a/src/extensions/storage.ts b/src/extensions/storage.ts index 8fad920a..9246e61e 100644 --- a/src/extensions/storage.ts +++ b/src/extensions/storage.ts @@ -3,7 +3,7 @@ import { StorageExtension } from '@posthog/plugin-scaffold' export function createStorage(server: PluginsServer, pluginConfig: PluginConfig): StorageExtension { const get = async function (key: string, defaultValue: unknown): Promise { - const result = await server.db.query( + const result = await server.db.postgresQuery( 'SELECT * FROM posthog_pluginstorage WHERE "plugin_config_id"=$1 AND "key"=$2 LIMIT 1', [pluginConfig.id, key] ) @@ -11,16 +11,16 @@ export function createStorage(server: PluginsServer, pluginConfig: PluginConfig) } const set = async function (key: string, value: unknown): Promise { if (typeof value === 'undefined') { - await server.db.query('DELETE FROM posthog_pluginstorage WHERE "plugin_config_id"=$1 AND "key"=$2', [ - pluginConfig.id, - key, - ]) + await server.db.postgresQuery( + 'DELETE FROM posthog_pluginstorage WHERE "plugin_config_id"=$1 AND "key"=$2', + [pluginConfig.id, key] + ) } else { - await server.db.query( + await server.db.postgresQuery( ` - INSERT INTO posthog_pluginstorage ("plugin_config_id", "key", "value") + INSERT INTO posthog_pluginstorage ("plugin_config_id", "key", "value") VALUES ($1, $2, $3) - ON CONFLICT ("plugin_config_id", "key") + ON CONFLICT ("plugin_config_id", "key") DO UPDATE SET value = $3 `, [pluginConfig.id, key, JSON.stringify(value)] diff --git a/src/idl/events.proto b/src/idl/events.proto new file mode 100644 index 00000000..afb1ce07 --- /dev/null +++ b/src/idl/events.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; +import "google/protobuf/timestamp.proto"; + +message Event { + string uuid = 1; + string event = 2; + string properties = 3; + string timestamp = 4; + uint64 team_id = 5; + string distinct_id = 6; + string created_at = 7; + string elements_chain = 8; + google.protobuf.Timestamp proto_created_at = 9; + google.protobuf.Timestamp proto_timestamp = 10; +} \ No newline at end of file diff --git a/src/idl/google/protobuf/timestamp.proto b/src/idl/google/protobuf/timestamp.proto new file mode 100644 index 00000000..9c4c75cc --- /dev/null +++ b/src/idl/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} \ No newline at end of file diff --git a/src/ingestion/kafka-queue.ts b/src/ingestion/kafka-queue.ts index 957b381a..7c1ba2a9 100644 --- a/src/ingestion/kafka-queue.ts +++ b/src/ingestion/kafka-queue.ts @@ -1,18 +1,16 @@ import * as Sentry from '@sentry/node' import { Kafka, Consumer, Message, EachBatchPayload } from 'kafkajs' import { PluginsServer, Queue, RawEventMessage } from 'types' -import { KAFKA_EVENTS_WAL } from './topics' import { PluginEvent } from '@posthog/plugin-scaffold' import { status } from '../status' - -export type BatchCallback = (messages: Message[]) => Promise +import { killGracefully } from '../utils' export class KafkaQueue implements Queue { private pluginsServer: PluginsServer private kafka: Kafka private consumer: Consumer private wasConsumerRan: boolean - private processEventBatch: (batch: PluginEvent[]) => Promise + private processEventBatch: (batch: PluginEvent[]) => Promise private saveEvent: (event: PluginEvent) => Promise constructor( @@ -33,27 +31,33 @@ export class KafkaQueue implements Queue { resolveOffset, heartbeat, commitOffsetsIfNecessary, - uncommittedOffsets, isRunning, isStale, }: EachBatchPayload): Promise { const batchProcessingTimer = new Date() - const rawEvents: RawEventMessage[] = batch.messages.map((message) => ({ - ...JSON.parse(message.value!.toString()), - kafka_offset: message.offset, - })) - const parsedEvents = rawEvents.map((rawEvent) => ({ - ...rawEvent, - data: JSON.parse(rawEvent.data), - })) - const pluginEvents: PluginEvent[] = parsedEvents.map((parsedEvent) => ({ - ...parsedEvent, - event: parsedEvent.data.event, - properties: parsedEvent.data.properties, - })) - const processedEvents: PluginEvent[] = ( - await this.processEventBatch(pluginEvents) - ).filter((event: PluginEvent[] | false | null | undefined) => Boolean(event)) + + const uuidOrder = new Map() + const uuidOffset = new Map() + const pluginEvents: PluginEvent[] = batch.messages.map((message, index) => { + const { data: dataStr, ...rawEvent } = JSON.parse(message.value!.toString()) + const event = { ...rawEvent, ...JSON.parse(dataStr) } + uuidOrder.set(event.uuid, index) + uuidOffset.set(event.uuid, message.offset) + return { + ...event, + site_url: event.site_url || null, + ip: event.ip || null, + } + }) + + const processedEvents = await this.processEventBatch(pluginEvents) + + // Sort in the original order that the events came in, putting any randomly added events to the end. + // This is so we would resolve the correct kafka offsets in order. + processedEvents.sort( + (a, b) => (uuidOrder.get(a.uuid!) || pluginEvents.length) - (uuidOrder.get(b.uuid!) || pluginEvents.length) + ) + for (const event of processedEvents) { if (!isRunning()) { status.info('😮', 'Consumer not running anymore, canceling batch processing!') @@ -65,12 +69,17 @@ export class KafkaQueue implements Queue { } const singleIngestionTimer = new Date() await this.saveEvent(event) - resolveOffset(event.kafka_offset!) + const offset = uuidOffset.get(event.uuid!) + if (offset) { + resolveOffset(offset) + } await heartbeat() await commitOffsetsIfNecessary() this.pluginsServer.statsd?.timing('kafka_queue.single_ingestion', singleIngestionTimer) } this.pluginsServer.statsd?.timing('kafka_queue.each_batch', batchProcessingTimer) + resolveOffset(batch.lastOffset()) + await commitOffsetsIfNecessary() } async start(): Promise { @@ -79,12 +88,10 @@ export class KafkaQueue implements Queue { this.consumer.on(this.consumer.events.CRASH, ({ payload: { error } }) => reject(error)) status.info('⏬', `Connecting Kafka consumer to ${this.pluginsServer.KAFKA_HOSTS}...`) this.wasConsumerRan = true - await this.consumer.subscribe({ topic: KAFKA_EVENTS_WAL }) + await this.consumer.subscribe({ topic: this.pluginsServer.KAFKA_CONSUMPTION_TOPIC! }) // KafkaJS batching: https://kafka.js.org/docs/consuming#a-name-each-batch-a-eachbatch await this.consumer.run({ - // TODO: eachBatchAutoResolve: false, // don't autoresolve whole batch in case we exit it early - // The issue is right now we'd miss some messages and not resolve them as processEventBatch COMPETELY - // discards some events, leaving us with no kafka_offset to resolve when in fact it should be resolved. + eachBatchAutoResolve: false, // we are resolving the last offset of the batch more deliberately autoCommitInterval: 500, // autocommit every 500 ms… autoCommitThreshold: 1000, // …or every 1000 messages, whichever is sooner eachBatch: this.eachBatch.bind(this), @@ -98,7 +105,7 @@ export class KafkaQueue implements Queue { return } status.info('⏳', 'Pausing Kafka consumer...') - await this.consumer.pause([{ topic: KAFKA_EVENTS_WAL }]) + await this.consumer.pause([{ topic: this.pluginsServer.KAFKA_CONSUMPTION_TOPIC! }]) status.info('⏸', 'Kafka consumer paused!') } @@ -107,12 +114,12 @@ export class KafkaQueue implements Queue { return } status.info('⏳', 'Resuming Kafka consumer...') - await this.consumer.resume([{ topic: KAFKA_EVENTS_WAL }]) + await this.consumer.resume([{ topic: this.pluginsServer.KAFKA_CONSUMPTION_TOPIC! }]) status.info('▶️', 'Kafka consumer resumed!') } isPaused(): boolean { - return this.consumer.paused().some(({ topic }) => topic === KAFKA_EVENTS_WAL) + return this.consumer.paused().some(({ topic }) => topic === this.pluginsServer.KAFKA_CONSUMPTION_TOPIC) } async stop(): Promise { @@ -130,7 +137,7 @@ export class KafkaQueue implements Queue { private static buildConsumer(kafka: Kafka): Consumer { const consumer = kafka.consumer({ - groupId: 'plugin-server', + groupId: 'clickhouse-ingestion', readUncommitted: false, }) const { GROUP_JOIN, CRASH, CONNECT, DISCONNECT } = consumer.events @@ -140,6 +147,7 @@ export class KafkaQueue implements Queue { consumer.on(CRASH, ({ payload: { error, groupId } }) => { status.error('⚠️', `Kafka consumer group ${groupId} crashed:\n`, error) Sentry.captureException(error) + killGracefully() }) consumer.on(CONNECT, () => { status.info('✅', 'Kafka consumer connected!') diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 1aef9775..55b7db7c 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,38 +1,533 @@ -import { DateTime } from 'luxon' -import { PluginsServer, Properties } from 'types' -import { UUIDT } from '../utils' -import { PluginEvent } from '@posthog/plugin-scaffold' +import { PluginEvent, Properties } from '@posthog/plugin-scaffold' +import { DateTime, Duration } from 'luxon' +import { + CohortPeople, + Element, + Person, + PersonDistinctId, + PluginsServer, + PostgresSessionRecordingEvent, + SessionRecordingEvent, + Team, + TimestampFormat, +} from '../types' +import { castTimestampOrNow, UUID, UUIDT } from '../utils' +import { Event as EventProto, IEvent } from '../idl/protos' +import { Producer } from 'kafkajs' +import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' +import { elementsToString, sanitizeEventName } from './utils' +import ClickHouse from '@posthog/clickhouse' +import { DB } from '../db' +import { status } from '../status' +import * as Sentry from '@sentry/node' +import { nodePostHog } from 'posthog-js-lite/dist/src/targets/node' +import Client from '../celery/client' export class EventsProcessor { pluginsServer: PluginsServer + db: DB + clickhouse: ClickHouse + kafkaProducer: Producer + celery: Client + posthog: ReturnType constructor(pluginsServer: PluginsServer) { this.pluginsServer = pluginsServer + this.db = pluginsServer.db + this.clickhouse = pluginsServer.clickhouse! + this.kafkaProducer = pluginsServer.kafkaProducer! + this.celery = new Client(pluginsServer.redis, pluginsServer.CELERY_DEFAULT_QUEUE) + this.posthog = nodePostHog('sTMFPsFhdP1Ssg') + if (process.env.NODE_ENV === 'test') { + this.posthog.optOut() + } } - async process_event_ee( - distinct_id: string, + public async processEvent( + distinctId: string, ip: string, - site_url: string, + siteUrl: string, data: PluginEvent, - team_id: number, + teamId: number, now: DateTime, - sent_at: DateTime | null - ): Promise { + sentAt: DateTime | null, + eventUuid: string + ): Promise { + if (!UUID.validateString(eventUuid, false)) { + throw new Error(`Not a valid UUID: "${eventUuid}"`) + } const singleSaveTimer = new Date() const properties: Properties = data.properties ?? {} if (data['$set']) { properties['$set'] = data['$set'] } + if (data['$set_once']) { + properties['$set_once'] = data['$set_once'] + } - const person_uuid = new UUIDT() - const event_uuid = new UUIDT() + const personUuid = new UUIDT().toString() + + const ts = this.handleTimestamp(data, now, sentAt) + await this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) + + let result: IEvent | SessionRecordingEvent if (data['event'] === '$snapshot') { + result = await this.createSessionRecordingEvent( + eventUuid, + teamId, + distinctId, + properties['$session_id'], + ts, + properties['$snapshot_data'] + ) this.pluginsServer.statsd?.timing('kafka_queue.single_save.snapshot', singleSaveTimer) } else { + result = await this.captureEE( + eventUuid, + personUuid, + ip, + siteUrl, + teamId, + data['event'], + distinctId, + properties, + ts, + sentAt + ) this.pluginsServer.statsd?.timing('kafka_queue.single_save.standard', singleSaveTimer) } + + return result + } + + private handleTimestamp(data: PluginEvent, now: DateTime, sentAt: DateTime | null): DateTime { + if (data['timestamp']) { + if (sentAt) { + // sent_at - timestamp == now - x + // x = now + (timestamp - sent_at) + try { + // timestamp and sent_at must both be in the same format: either both with or both without timezones + // otherwise we can't get a diff to add to now + return now.plus(DateTime.fromISO(data['timestamp']).diff(sentAt)) + } catch (error) { + status.error('⚠️', 'Error when handling timestamp:', error) + Sentry.captureException(error) + } + } + return DateTime.fromISO(data['timestamp']) + } + if (data['offset']) { + return now.minus(Duration.fromMillis(data['offset'])) + } + return now + } + + private async handleIdentifyOrAlias( + event: string, + properties: Properties, + distinctId: string, + teamId: number + ): Promise { + if (event === '$create_alias') { + await this.alias(properties['alias'], distinctId, teamId) + } else if (event === '$identify') { + if (properties['$anon_distinct_id']) { + await this.alias(properties['$anon_distinct_id'], distinctId, teamId) + } + if (properties['$set'] || properties['$set_once']) { + await this.updatePersonProperties( + teamId, + distinctId, + properties['$set'] || {}, + properties['$set_once'] || {} + ) + } + await this.setIsIdentified(teamId, distinctId) + } + } + + private async setIsIdentified(teamId: number, distinctId: string, isIdentified = true): Promise { + let personFound = await this.db.fetchPerson(teamId, distinctId) + if (!personFound) { + try { + const personCreated = await this.db.createPerson( + DateTime.utc(), + {}, + teamId, + null, + true, + new UUIDT().toString() + ) + await this.db.addDistinctId(personCreated, distinctId) + } catch { + // Catch race condition where in between getting and creating, + // another request already created this person + personFound = await this.db.fetchPerson(teamId, distinctId) + } + } + if (personFound && !personFound.is_identified) { + await this.db.updatePerson(personFound, { is_identified: isIdentified }) + } + } + + private async updatePersonProperties( + teamId: number, + distinctId: string, + properties: Properties, + propertiesOnce: Properties + ): Promise { + let personFound = await this.db.fetchPerson(teamId, distinctId) + if (!personFound) { + try { + const personCreated = await this.db.createPerson( + DateTime.utc(), + properties, + teamId, + null, + false, + new UUIDT().toString() + ) + await this.db.addDistinctId(personCreated, distinctId) + } catch { + // Catch race condition where in between getting and creating, + // another request already created this person + personFound = await this.db.fetchPerson(teamId, distinctId) + } + } + const updatedProperties: Properties = { ...propertiesOnce, ...personFound!.properties, ...properties } + return await this.db.updatePerson(personFound!, { properties: updatedProperties }) + } + + private async alias( + previousDistinctId: string, + distinctId: string, + teamId: number, + retryIfFailed = true + ): Promise { + const oldPerson = await this.db.fetchPerson(teamId, previousDistinctId) + const newPerson = await this.db.fetchPerson(teamId, distinctId) + + if (oldPerson && !newPerson) { + try { + await this.db.addDistinctId(oldPerson, distinctId) + // Catch race case when somebody already added this distinct_id between .get and .addDistinctId + } catch { + // integrity error + if (retryIfFailed) { + // run everything again to merge the users if needed + await this.alias(previousDistinctId, distinctId, teamId, false) + } + } + return + } + + if (!oldPerson && newPerson) { + try { + await this.db.addDistinctId(newPerson, previousDistinctId) + // Catch race case when somebody already added this distinct_id between .get and .addDistinctId + } catch { + // integrity error + if (retryIfFailed) { + // run everything again to merge the users if needed + await this.alias(previousDistinctId, distinctId, teamId, false) + } + } + return + } + + if (!oldPerson && !newPerson) { + try { + const personCreated = await this.db.createPerson( + DateTime.utc(), + {}, + teamId, + null, + false, + new UUIDT().toString() + ) + await this.db.addDistinctId(personCreated, distinctId) + await this.db.addDistinctId(personCreated, previousDistinctId) + } catch { + // Catch race condition where in between getting and creating, + // another request already created this person + if (retryIfFailed) { + // Try once more, probably one of the two persons exists now + await this.alias(previousDistinctId, distinctId, teamId, false) + } + } + return + } + + if (oldPerson && newPerson && oldPerson.id !== newPerson.id) { + await this.mergePeople(newPerson, [oldPerson]) + } + } + + public async mergePeople(mergeInto: Person, peopleToMerge: Person[]): Promise { + let firstSeen = mergeInto.created_at + + // merge the properties + for (const otherPerson of peopleToMerge) { + mergeInto.properties = { ...otherPerson.properties, ...mergeInto.properties } + if (otherPerson.created_at < firstSeen) { + // Keep the oldest created_at (i.e. the first time we've seen this person) + firstSeen = otherPerson.created_at + } + } + + await this.db.updatePerson(mergeInto, { created_at: firstSeen, properties: mergeInto.properties }) + + // merge the distinct_ids + for (const otherPerson of peopleToMerge) { + const otherPersonDistinctIds: PersonDistinctId[] = ( + await this.db.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', + [otherPerson.id, mergeInto.team_id] + ) + ).rows + for (const personDistinctId of otherPersonDistinctIds) { + await this.db.moveDistinctId(otherPerson, personDistinctId, mergeInto) + } + + await this.db.postgresQuery('UPDATE posthog_cohortpeople SET person_id = $1 WHERE person_id = $2', [ + mergeInto.id, + otherPerson.id, + ]) + + await this.db.deletePerson(otherPerson) + } + } + + private async captureEE( + eventUuid: string, + personUuid: string, + ip: string, + siteUrl: string, + teamId: number, + event: string, + distinctId: string, + properties: Properties, + timestamp: DateTime, + sentAt: DateTime | null + ): Promise { + event = sanitizeEventName(event) + + const elements: Record[] | undefined = properties['$elements'] + let elementsList: Element[] = [] + if (elements && elements.length) { + delete properties['$elements'] + elementsList = elements.map((el) => ({ + text: el['$el_text']?.slice(0, 400), + tag_name: el['tag_name'], + href: el['attr__href']?.slice(0, 2048), + attr_class: el['attr__class']?.split(' '), + attr_id: el['attr__id'], + nth_child: el['nth_child'], + nth_of_type: el['nth_of_type'], + attributes: Object.fromEntries(Object.entries(el).filter(([key]) => key.startsWith('attr__'))), + })) + } + + const teamQueryResult = await this.db.postgresQuery('SELECT * FROM posthog_team WHERE id = $1', [teamId]) + const team: Team = teamQueryResult.rows[0] + + if (!team.anonymize_ips && !('$ip' in properties)) { + properties['$ip'] = ip + } + + await this.storeNamesAndProperties(team, event, properties) + + const pdiSelectResult = await this.db.postgresQuery( + 'SELECT COUNT(*) AS pdicount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', + [teamId, distinctId] + ) + const pdiCount = parseInt(pdiSelectResult.rows[0].pdicount) + + if (!pdiCount) { + // Catch race condition where in between getting and creating, another request already created this user + try { + const personCreated: Person = await this.db.createPerson( + sentAt || DateTime.utc(), + {}, + teamId, + null, + false, + personUuid.toString(), + [distinctId] + ) + } catch {} + } + + return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList, siteUrl) + } + + private async storeNamesAndProperties(team: Team, event: string, properties: Properties): Promise { + // In _capture we only prefetch a couple of fields in Team to avoid fetching too much data + let save = false + if (!team.ingested_event) { + // First event for the team captured + const organizationMembers = await this.db.postgresQuery( + 'SELECT distinct_id FROM posthog_user JOIN posthog_organizationmembership ON posthog_user.id = posthog_organizationmembership.user_id WHERE organization_id = $1', + [team.organization_id] + ) + const distinctIds: { distinct_id: string }[] = (await organizationMembers).rows + for (const { distinct_id } of distinctIds) { + this.posthog.identify(distinct_id) + this.posthog.capture('first team event ingested', { team: team.uuid }) + } + team.ingested_event = true + save = true + } + if (team.event_names && !team.event_names.includes(event)) { + save = true + team.event_names.push(event) + team.event_names_with_usage.push({ event: event, usage_count: null, volume: null }) + } + for (const [key, value] of Object.entries(properties)) { + if (team.event_properties && !team.event_properties.includes(key)) { + team.event_properties.push(key) + team.event_properties_with_usage.push({ key: key, usage_count: null, volume: null }) + save = true + } + if ( + typeof value === 'number' && + team.event_properties_numerical && + !team.event_properties_numerical.includes(key) + ) { + team.event_properties_numerical.push(key) + save = true + } + } + if (save) { + await this.db.postgresQuery( + `UPDATE posthog_team SET + ingested_event = $1, event_names = $2, event_names_with_usage = $3, event_properties = $4, + event_properties_with_usage = $5, event_properties_numerical = $6 + WHERE id = $7`, + [ + team.ingested_event, + JSON.stringify(team.event_names), + JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties), + JSON.stringify(team.event_properties_with_usage), + JSON.stringify(team.event_properties_numerical), + team.id, + ] + ) + } + } + + private async createEvent( + uuid: string, + event: string, + team: Team, + distinctId: string, + properties?: Properties, + timestamp?: DateTime | string, + elements?: Element[], + siteUrl?: string + ): Promise { + const timestampString = castTimestampOrNow( + timestamp, + this.kafkaProducer ? TimestampFormat.ClickHouse : TimestampFormat.ISO + ) + const elementsChain = elements && elements.length ? elementsToString(elements) : '' + + const data: IEvent = { + uuid, + event, + properties: JSON.stringify(properties ?? {}), + timestamp: timestampString, + teamId: team.id, + distinctId, + elementsChain, + createdAt: timestampString, + } + + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_EVENTS, + messages: [ + { + key: uuid, + value: EventProto.encodeDelimited(EventProto.create(data)).finish() as Buffer, + }, + ], + }) + } else { + let elementsHash = '' + if (elements && elements.length > 0) { + elementsHash = await this.db.createElementGroup(elements, team.id) + } + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_event (created_at, event, distinct_id, properties, team_id, timestamp, elements, elements_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *', + [ + data.createdAt, + data.event, + distinctId, + data.properties, + data.teamId, + data.timestamp, + JSON.stringify(elements || []), + elementsHash, + ] + ) + const eventCreated = insertResult.rows[0] as Event + } + + this.celery.sendTask('ee.tasks.webhooks_ee.post_event_to_webhook_ee', [ + { + event, + properties, + distinct_id: distinctId, + timestamp, + elements_list: elements, + }, + team.id, + siteUrl, + ]) + + return data + } + + private async createSessionRecordingEvent( + uuid: string, + team_id: number, + distinct_id: string, + session_id: string, + timestamp: DateTime | string, + snapshot_data: Record + ): Promise { + const timestampString = castTimestampOrNow( + timestamp, + this.kafkaProducer ? TimestampFormat.ClickHouse : TimestampFormat.ISO + ) + + const data: SessionRecordingEvent = { + uuid, + team_id: team_id, + distinct_id: distinct_id, + session_id: session_id, + snapshot_data: JSON.stringify(snapshot_data), + timestamp: timestampString, + created_at: timestampString, + } + + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_SESSION_RECORDING_EVENTS, + messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }], + }) + } else { + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_sessionrecordingevent (created_at, team_id, distinct_id, session_id, timestamp, snapshot_data) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', + [data.created_at, data.team_id, data.distinct_id, data.session_id, data.timestamp, data.snapshot_data] + ) + const eventCreated = insertResult.rows[0] as PostgresSessionRecordingEvent + return eventCreated + } + return data } } diff --git a/src/ingestion/topics.ts b/src/ingestion/topics.ts index c90ebf55..a76767cb 100644 --- a/src/ingestion/topics.ts +++ b/src/ingestion/topics.ts @@ -3,3 +3,4 @@ export const KAFKA_PERSON = 'clickhouse_person' export const KAFKA_PERSON_UNIQUE_ID = 'clickhouse_person_unique_id' export const KAFKA_SESSION_RECORDING_EVENTS = 'clickhouse_session_recording_events' export const KAFKA_EVENTS_WAL = 'events_write_ahead_log' +export const KAFKA_EVENTS_PLUGIN_INGESTION = 'events_plugin_ingestion' diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts new file mode 100644 index 00000000..ba46ab3b --- /dev/null +++ b/src/ingestion/utils.ts @@ -0,0 +1,155 @@ +import { Element, BasePerson, RawPerson, Person } from '../types' +import crypto from 'crypto' + +export function unparsePersonPartial(person: Partial): Partial { + return { ...(person as BasePerson), ...(person.created_at ? { created_at: person.created_at.toISO() } : {}) } +} + +export function escapeQuotes(input: string): string { + return input.replace(/"/g, '\\"') +} + +export function elementsToString(elements: Element[]): string { + const ret = elements.map((element) => { + let el_string = '' + if (element.tag_name) { + el_string += element.tag_name + } + if (element.attr_class) { + element.attr_class.sort() + for (const single_class of element.attr_class) { + el_string += `.${single_class.replace(/"/g, '')}` + } + } + let attributes: Record = { + ...(element.text ? { text: element.text } : {}), + 'nth-child': element.nth_child ?? 0, + 'nth-of-type': element.nth_of_type ?? 0, + ...(element.href ? { href: element.href } : {}), + ...(element.attr_id ? { attr_id: element.attr_id } : {}), + ...element.attributes, + } + attributes = Object.fromEntries( + Object.entries(attributes) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => [escapeQuotes(key.toString()), escapeQuotes(value.toString())]) + ) + el_string += ':' + el_string += Object.entries(attributes) + .map(([key, value]) => `${key}="${value}"`) + .join('') + return el_string + }) + return ret.join(';') +} + +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export function sanitizeEventName(eventName: any): string { + if (typeof eventName !== 'string') { + try { + eventName = JSON.stringify(eventName) + } catch { + eventName = String(eventName) + } + } + return eventName.substr(0, 200) +} + +/** Escape UTF-8 characters into `\u1234`. */ +function jsonEscapeUtf8(s: string): string { + return s.replace(/[^\x20-\x7F]/g, (x) => '\\u' + ('000' + x.codePointAt(0)?.toString(16)).slice(-4)) +} + +/** Produce output compatible with that of Python's `json.dumps`. */ +function jsonDumps(obj: any): string { + if (typeof obj === 'object' && obj !== null) { + if (Array.isArray(obj)) { + return `[${obj.map(jsonDumps).join(', ')}]` // space after comma + } else { + return `{${Object.keys(obj) // no space after '{' or before '}' + .sort() // must sort the keys of the object! + .map((k) => `${jsonDumps(k)}: ${jsonDumps(obj[k])}`) // space after ':' + .join(', ')}}` // space after ',' + } + } else if (typeof obj === 'string') { + return jsonEscapeUtf8(JSON.stringify(obj)) + } else { + return JSON.stringify(obj) + } +} + +export function hashElements(elements: Element[]): string { + const elementsList = elements.map((element) => ({ + attributes: element.attributes ?? null, + text: element.text ?? null, + tag_name: element.tag_name ?? null, + href: element.href ?? null, + attr_id: element.attr_id ?? null, + attr_class: element.attr_class ?? null, + nth_child: element.nth_child ?? null, + nth_of_type: element.nth_of_type ?? null, + order: element.order ?? null, + })) + + const serializedString = jsonDumps(elementsList) + + return crypto.createHash('md5').update(serializedString).digest('hex') +} + +export function chainToElements(chain: string): Element[] { + const elements: Element[] = [] + + // Below splits all elements by ;, while ignoring escaped quotes and semicolons within quotes + const splitChainRegex = /(?:[^\s;"]|"(?:\\.|[^"])*")+/g + + // Below splits the tag/classes from attributes + // Needs a regex because classes can have : too + const splitClassAttributes = /(.*?)($|:([a-zA-Z\-_0-9]*=.*))/g + const parseAttributesRegex = /((.*?)="(.*?[^\\])")/gm + + Array.from(chain.matchAll(splitChainRegex)) + .map((r) => r[0]) + .forEach((elString, index) => { + const elStringSplit = Array.from(elString.matchAll(splitClassAttributes))[0] + const attributes = + elStringSplit.length > 3 + ? Array.from(elStringSplit[3].matchAll(parseAttributesRegex)).map((a) => [a[2], a[3]]) + : [] + + const element: Element = { + attributes: {}, + order: index, + } + + if (elStringSplit[1]) { + const tagAndClass = elStringSplit[1].split('.') + element.tag_name = tagAndClass[0] + if (tagAndClass.length > 1) { + const [_, ...rest] = tagAndClass + element.attr_class = rest.filter((t) => t) + } + } + + for (const [key, value] of attributes) { + if (key == 'href') { + element.href = value + } else if (key == 'nth-child') { + element.nth_child = parseInt(value) + } else if (key == 'nth-of-type') { + element.nth_of_type = parseInt(value) + } else if (key == 'text') { + element.text = value + } else if (key == 'attr_id') { + element.attr_id = value + } else if (key) { + if (!element.attributes) { + element.attributes = {} + } + element.attributes[key] = value + } + } + elements.push(element) + }) + + return elements +} diff --git a/src/plugins.ts b/src/plugins.ts index 95ffb91e..b80976e8 100644 --- a/src/plugins.ts +++ b/src/plugins.ts @@ -264,7 +264,7 @@ export async function runPluginsOnBatch(server: PluginsServer, batch: PluginEven allReturnedEvents = allReturnedEvents.concat(returnedEvents) } - return allReturnedEvents + return allReturnedEvents.filter(Boolean) } export async function runPluginTask(server: PluginsServer, taskName: string, pluginConfigId: number): Promise { diff --git a/src/server.ts b/src/server.ts index 5c0f20fc..a17d2a18 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,10 +1,11 @@ -import { Pool } from 'pg' +import { Pool, types as pgTypes } from 'pg' import * as schedule from 'node-schedule' import Redis from 'ioredis' -import { Kafka, logLevel } from 'kafkajs' +import { Kafka, logLevel, Producer } from 'kafkajs' import { FastifyInstance } from 'fastify' import { PluginsServer, PluginsServerConfig, Queue } from './types' import { startQueue } from './worker/queue' +import ClickHouse from '@posthog/clickhouse' import { startFastifyInstance, stopFastifyInstance } from './web/server' import { version } from '../package.json' import { PluginEvent } from '@posthog/plugin-scaffold' @@ -17,6 +18,10 @@ import { EventsProcessor } from './ingestion/process-event' import { status } from './status' import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' +import { DB } from './db' +import { DateTime } from 'luxon' +import * as fs from 'fs' +import { KAFKA_EVENTS_PLUGIN_INGESTION, KAFKA_EVENTS_WAL } from './ingestion/topics' export async function createServer( config: Partial = {}, @@ -40,15 +45,6 @@ export async function createServer( }) await redis.info() - const db = new Pool({ - connectionString: serverConfig.DATABASE_URL, - ssl: process.env.DEPLOYMENT?.startsWith('Heroku') - ? { - rejectUnauthorized: false, - } - : undefined, - }) - let kafkaSsl: ConnectionOptions | undefined if ( serverConfig.KAFKA_CLIENT_CERT_B64 && @@ -68,19 +64,68 @@ export async function createServer( } } + let clickhouse: ClickHouse | undefined let kafka: Kafka | undefined + let kafkaProducer: Producer | undefined if (serverConfig.KAFKA_ENABLED) { if (!serverConfig.KAFKA_HOSTS) { throw new Error('You must set KAFKA_HOSTS to process events from Kafka!') } + clickhouse = new ClickHouse({ + host: serverConfig.CLICKHOUSE_HOST, + port: serverConfig.CLICKHOUSE_SECURE ? 8443 : 8123, + protocol: serverConfig.CLICKHOUSE_SECURE ? 'https:' : 'http:', + user: serverConfig.CLICKHOUSE_USER, + password: serverConfig.CLICKHOUSE_PASSWORD || undefined, + dataObjects: true, + queryOptions: { + database: serverConfig.CLICKHOUSE_DATABASE, + output_format_json_quote_64bit_integers: false, + }, + ca: serverConfig.CLICKHOUSE_CA ? fs.readFileSync(serverConfig.CLICKHOUSE_CA).toString() : undefined, + }) + await clickhouse.querying('SELECT 1') // test that the connection works + + if (!serverConfig.KAFKA_CONSUMPTION_TOPIC) { + // When ingesting events, listen to the "INGESTION" topic, otherwise listen to the "WAL" and discard + serverConfig.KAFKA_CONSUMPTION_TOPIC = serverConfig.PLUGIN_SERVER_INGESTION + ? KAFKA_EVENTS_PLUGIN_INGESTION + : KAFKA_EVENTS_WAL + } + kafka = new Kafka({ clientId: `plugin-server-v${version}-${new UUIDT()}`, brokers: serverConfig.KAFKA_HOSTS.split(','), - logLevel: logLevel.NOTHING, + logLevel: logLevel.WARN, ssl: kafkaSsl, }) + kafkaProducer = kafka.producer() + await kafkaProducer?.connect() } + // `node-postgres` will return dates as plain JS Date objects, which will use the local timezone. + // This converts all date fields to a proper luxon UTC DateTime and then casts them to a string + // Unfortunately this must be done on a global object before initializing the `Pool` + pgTypes.setTypeParser(1083 /* types.TypeId.TIME */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + pgTypes.setTypeParser(1114 /* types.TypeId.TIMESTAMP */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + pgTypes.setTypeParser(1184 /* types.TypeId.TIMESTAMPTZ */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + + const postgres = new Pool({ + connectionString: serverConfig.DATABASE_URL, + ssl: process.env.DEPLOYMENT?.startsWith('Heroku') + ? { + rejectUnauthorized: false, + } + : undefined, + }) + const db = new DB(postgres, kafkaProducer, clickhouse) + let statsd: StatsD | undefined if (serverConfig.STATSD_HOST) { statsd = new StatsD({ @@ -100,8 +145,11 @@ export async function createServer( const server: Omit = { ...serverConfig, db, + postgres, redis, + clickhouse, kafka, + kafkaProducer, statsd, plugins: new Map(), pluginConfigs: new Map(), @@ -115,8 +163,9 @@ export async function createServer( server.eventsProcessor = new EventsProcessor(server as PluginsServer) const closeServer = async () => { + await kafkaProducer?.disconnect() await server.redis.quit() - await server.db.end() + await server.postgres.end() } return [server as PluginsServer, closeServer] @@ -165,7 +214,7 @@ export async function startPluginsServer( await stopFastifyInstance(fastifyInstance!) } await queue?.stop() - pubSub?.disconnect() + await pubSub?.quit() pingJob && schedule.cancelJob(pingJob) statsJob && schedule.cancelJob(statsJob) await stopSchedule?.() @@ -207,6 +256,7 @@ export async function startPluginsServer( fastifyInstance = await startFastifyInstance(server) } + stopSchedule = await startSchedule(server, piscina) queue = await startQueue(server, processEvent, processEventBatch) piscina.on('drain', () => { queue?.resume() @@ -243,8 +293,6 @@ export async function startPluginsServer( } }) - stopSchedule = await startSchedule(server, piscina) - status.info('🚀', 'All systems go.') } catch (error) { Sentry.captureException(error) diff --git a/src/services/schedule.ts b/src/services/schedule.ts index 9a2db4f9..219ca177 100644 --- a/src/services/schedule.ts +++ b/src/services/schedule.ts @@ -13,7 +13,7 @@ export async function startSchedule( piscina: Piscina, onLock?: () => void ): Promise<() => Promise> { - status.info('⏰', 'Starting scheduling service') + status.info('⏰', 'Starting scheduling service...') let stopped = false let weHaveTheLock = false diff --git a/src/sql.ts b/src/sql.ts index 78b262b2..c247a3e0 100644 --- a/src/sql.ts +++ b/src/sql.ts @@ -1,7 +1,7 @@ import { Plugin, PluginAttachmentDB, PluginConfig, PluginConfigId, PluginError, PluginsServer } from './types' export async function getPluginRows(server: PluginsServer): Promise { - const { rows: pluginRows }: { rows: Plugin[] } = await server.db.query( + const { rows: pluginRows }: { rows: Plugin[] } = await server.db.postgresQuery( `SELECT posthog_plugin.* FROM posthog_plugin WHERE id in (SELECT posthog_pluginconfig.plugin_id FROM posthog_pluginconfig @@ -13,7 +13,7 @@ export async function getPluginRows(server: PluginsServer): Promise { } export async function getPluginAttachmentRows(server: PluginsServer): Promise { - const { rows }: { rows: PluginAttachmentDB[] } = await server.db.query( + const { rows }: { rows: PluginAttachmentDB[] } = await server.db.postgresQuery( `SELECT posthog_pluginattachment.* FROM posthog_pluginattachment WHERE plugin_config_id in (SELECT posthog_pluginconfig.id FROM posthog_pluginconfig @@ -24,7 +24,7 @@ export async function getPluginAttachmentRows(server: PluginsServer): Promise { - const { rows }: { rows: PluginConfig[] } = await server.db.query( + const { rows }: { rows: PluginConfig[] } = await server.db.postgresQuery( `SELECT posthog_pluginconfig.* FROM posthog_pluginconfig LEFT JOIN posthog_team ON posthog_team.id = posthog_pluginconfig.team_id @@ -38,7 +38,7 @@ export async function setError( pluginError: PluginError | null, pluginConfig: PluginConfig | PluginConfigId ): Promise { - await server.db.query('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ + await server.db.postgresQuery('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ pluginError, typeof pluginConfig === 'object' ? pluginConfig?.id : pluginConfig, ]) diff --git a/src/types.ts b/src/types.ts index 2d8a48d9..d8bc7a53 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,12 +1,13 @@ import { Pool } from 'pg' import { Redis } from 'ioredis' -import { Kafka } from 'kafkajs' -import { PluginEvent, PluginAttachment, PluginConfigSchema } from '@posthog/plugin-scaffold' +import { Kafka, Producer } from 'kafkajs' +import { PluginEvent, PluginAttachment, PluginConfigSchema, Properties } from '@posthog/plugin-scaffold' import { VM } from 'vm2' import { DateTime } from 'luxon' import { StatsD } from 'hot-shots' import { EventsProcessor } from 'ingestion/process-event' -import { UUID } from './utils' +import ClickHouse from '@posthog/clickhouse' +import { DB } from './db' export enum LogLevel { Debug = 'debug', @@ -22,11 +23,18 @@ export interface PluginsServerConfig extends Record { TASKS_PER_WORKER: number CELERY_DEFAULT_QUEUE: string DATABASE_URL: string + CLICKHOUSE_HOST: string + CLICKHOUSE_DATABASE: string + CLICKHOUSE_USER: string + CLICKHOUSE_PASSWORD: string | null + CLICKHOUSE_CA: string | null + CLICKHOUSE_SECURE: boolean KAFKA_ENABLED: boolean KAFKA_HOSTS: string | null KAFKA_CLIENT_CERT_B64: string | null KAFKA_CLIENT_CERT_KEY_B64: string | null KAFKA_TRUSTED_CERT_B64: string | null + KAFKA_CONSUMPTION_TOPIC: string | null PLUGINS_CELERY_QUEUE: string REDIS_URL: string BASE_DIR: string @@ -35,6 +43,7 @@ export interface PluginsServerConfig extends Record { WEB_PORT: number WEB_HOSTNAME: string LOG_LEVEL: LogLevel + PLUGIN_SERVER_INGESTION: boolean SENTRY_DSN: string | null STATSD_HOST: string | null STATSD_PORT: number @@ -43,11 +52,14 @@ export interface PluginsServerConfig extends Record { } export interface PluginsServer extends PluginsServerConfig { - // active connections to Postgres, Redis, Kafka, StatsD - db: Pool + // active connections to Postgres, Redis, ClickHouse, Kafka, StatsD + db: DB + postgres: Pool redis: Redis - kafka: Kafka | undefined - statsd: StatsD | undefined + clickhouse?: ClickHouse + kafka?: Kafka + kafkaProducer?: Producer + statsd?: StatsD // currently enabled plugin status plugins: Map pluginConfigs: Map @@ -69,6 +81,10 @@ export interface Queue extends Pausable { stop: () => void } +export interface Queue { + stop: () => void +} + export type PluginId = number export type PluginConfigId = number export type TeamId = number @@ -145,26 +161,188 @@ export interface PluginConfigVMReponse { tasks: Record } -// received via Kafka -interface EventMessage { +export interface EventUsage { + event: string + usage_count: number | null + volume: number | null +} + +export interface PropertyUsage { + key: string + usage_count: number | null + volume: number | null +} + +/** Properties shared by RawEventMessage and EventMessage. */ +export interface BaseEventMessage { distinct_id: string ip: string site_url: string team_id: number + uuid: string } -export interface RawEventMessage extends EventMessage { +/** Raw event message as received via Kafka. */ +export interface RawEventMessage extends BaseEventMessage { + /** JSON-encoded object. */ data: string + /** ISO-formatted datetime. */ now: string - sent_at: string // may be an empty string - uuid: string + /** ISO-formatted datetime. May be empty! */ + sent_at: string + /** JSON-encoded number. */ + kafka_offset: string } -export interface ParsedEventMessage extends EventMessage { +/** Usable event message. */ +export interface EventMessage extends BaseEventMessage { data: PluginEvent now: DateTime sent_at: DateTime | null - uuid: UUID } -export type Properties = Record +/** Raw Organization row from database. */ +export interface RawOrganization { + id: string + name: string + created_at: string + updated_at: string +} + +/** Usable Team model. */ +export interface Team { + id: number + uuid: string + organization_id: string + name: string + anonymize_ips: boolean + api_token: string + app_urls: string[] + completed_snippet_onboarding: boolean + event_names: string[] + event_properties: string[] + event_properties_numerical: string[] + event_names_with_usage: EventUsage[] + event_properties_with_usage: PropertyUsage[] + opt_out_capture: boolean + slack_incoming_webhook: string + session_recording_opt_in: boolean + plugins_opt_in: boolean + ingested_event: boolean +} + +/** Usable Element model. */ +export interface Element { + text?: string + tag_name?: string + href?: string + attr_id?: string + attr_class?: string[] + nth_child?: number + nth_of_type?: number + attributes?: Record + event_id?: number + order?: number + group_id?: number +} + +export interface ElementGroup { + id: number + hash: string + team_id: number +} + +/** Usable Event model. */ +export interface Event { + id: number + event?: string + properties: Record + elements?: Element[] + timestamp: string + team_id: number + distinct_id: string + elements_hash: string + created_at: string +} + +export interface ClickHouseEvent extends Omit { + uuid: string + elements_chain: string +} + +/** Properties shared by RawPerson and Person. */ +export interface BasePerson { + id: number + team_id: number + properties: Properties + is_user_id: number + is_identified: boolean + uuid: string +} + +/** Raw Person row from database. */ +export interface RawPerson extends BasePerson { + created_at: string +} + +/** Usable Person model. */ +export interface Person extends BasePerson { + created_at: DateTime +} + +/** Clickhouse Person model. */ +export interface ClickHousePerson { + id: string + created_at: string + team_id: number + properties: string + is_identified: number + timestamp: string +} + +/** Usable PersonDistinctId model. */ +export interface PersonDistinctId { + id: number + team_id: number + person_id: number + distinct_id: string +} + +/** ClickHouse PersonDistinctId model. */ +export interface ClickHousePersonDistinctId { + id: number + team_id: number + person_id: string + distinct_id: string +} + +/** Usable CohortPeople model. */ +export interface CohortPeople { + id: number + cohort_id: number + person_id: number +} + +export interface SessionRecordingEvent { + uuid: string + timestamp: string + team_id: number + distinct_id: string + session_id: string + snapshot_data: string + created_at: string +} + +export interface PostgresSessionRecordingEvent extends Omit { + id: string +} + +export enum TimestampFormat { + ClickHouse = 'clickhouse', + ISO = 'iso', +} + +export enum Database { + ClickHouse = 'clickhouse', + Postgres = 'postgres', +} diff --git a/src/utils.ts b/src/utils.ts index 301adeaf..bd6227a4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,9 +2,22 @@ import { Readable } from 'stream' import * as tar from 'tar-stream' import AdmZip from 'adm-zip' import * as zlib from 'zlib' -import { LogLevel } from './types' +import { LogLevel, TimestampFormat } from './types' import { randomBytes } from 'crypto' import { DateTime } from 'luxon' +import { status } from './status' + +/** Time until autoexit (due to error) gives up on graceful exit and kills the process right away. */ +const GRACEFUL_EXIT_PERIOD_SECONDS = 5 + +export function killGracefully(): void { + status.error('⏲', 'Shutting plugin server down gracefully with SIGTERM...') + process.kill(process.pid, 'SIGTERM') + setTimeout(() => { + status.error('⏲', `Plugin server still running after ${GRACEFUL_EXIT_PERIOD_SECONDS} s, killing it forcefully!`) + process.exit(1) + }, GRACEFUL_EXIT_PERIOD_SECONDS * 1000) +} /** * @param binary Buffer @@ -134,12 +147,24 @@ for (let i = 0; i < 256; i++) { } export class UUID { - static validateString(candidate: string): void { - if (!candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i)) { + /** + * Check whether str + * + * This does not care about RFC4122, since neither does UUIDT above. + * https://stackoverflow.com/questions/7905929/how-to-test-valid-uuid-guid + */ + static validateString(candidate: any, throwOnInvalid = true): boolean { + const isValid = Boolean( + candidate && + typeof candidate === 'string' && + candidate.match(/^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$/i) + ) + if (!isValid && throwOnInvalid) { throw new Error( 'String does not match format XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX (where each X is a hexadecimal character)!' ) } + return isValid } array: Uint8Array @@ -266,8 +291,44 @@ export class UUIDT extends UUID { } } +/** Format timestamp for ClickHouse. */ +export function castTimestampOrNow( + timestamp?: DateTime | string | null, + timestampFormat: TimestampFormat = TimestampFormat.ISO +): string { + if (!timestamp) { + timestamp = DateTime.utc() + } else if (typeof timestamp === 'string') { + timestamp = DateTime.fromISO(timestamp) + } + timestamp = timestamp.toUTC() + if (timestampFormat === TimestampFormat.ClickHouse) { + return timestamp.toFormat('yyyy-MM-dd HH:mm:ss.u') + } else if (timestampFormat === TimestampFormat.ISO) { + return timestamp.toUTC().toISO() + } else { + throw new Error(`Unrecognized timestamp format ${timestampFormat}!`) + } +} + +export function clickHouseTimestampToISO(timestamp: string): string { + return DateTime.fromFormat(timestamp, 'yyyy-MM-dd HH:mm:ss.u', { zone: 'UTC' }).toISO() +} + export function delay(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, ms) }) } + +/** Remove all quotes from the provided identifier to prevent SQL injection. */ +export function sanitizeSqlIdentifier(unquotedIdentifier: string): string { + return unquotedIdentifier.replace(/[^\w\d_]+/g, '') +} + +/** Escape single quotes and slashes */ +export function escapeClickHouseString(string: string): string { + // In string literals, you need to escape at least `'` and `\`. + // https://clickhouse.tech/docs/en/sql-reference/syntax/ + return string.replace(/\\/g, '\\\\').replace(/'/g, "\\'") +} diff --git a/src/worker/piscina.js b/src/worker/piscina.js index 017ca080..016c90ff 100644 --- a/src/worker/piscina.js +++ b/src/worker/piscina.js @@ -9,7 +9,7 @@ if (isMainThread) { const piscina = new Piscina(createConfig(serverConfig, __filename)) piscina.on('error', (error) => { Sentry.captureException(error) - console.error('⚠️ Piscina worker thread error:\n', error) + console.error('⚠️', 'Piscina worker thread error:\n', error) }) return piscina }, diff --git a/src/worker/queue.ts b/src/worker/queue.ts index c3bdd808..4cf943ab 100644 --- a/src/worker/queue.ts +++ b/src/worker/queue.ts @@ -4,8 +4,9 @@ import { DateTime } from 'luxon' import Worker from '../celery/worker' import Client from '../celery/client' import { PluginsServer, Queue } from '../types' -import { status } from '../status' import { KafkaQueue } from '../ingestion/kafka-queue' +import { status } from '../status' +import { UUIDT } from '../utils' export async function startQueue( server: PluginsServer, @@ -45,15 +46,29 @@ async function startQueueRedis( const processedEvent = await processEvent(event) if (processedEvent) { const { distinct_id, ip, site_url, team_id, now, sent_at, ...data } = processedEvent - client.sendTask('posthog.tasks.process_event.process_event', [], { - distinct_id, - ip, - site_url, - data, - team_id, - now, - sent_at, - }) + + if (server.PLUGIN_SERVER_INGESTION) { + await server.eventsProcessor.processEvent( + distinct_id, + ip, + site_url, + processedEvent, + team_id, + DateTime.fromISO(now), + sent_at ? DateTime.fromISO(sent_at) : null, + new UUIDT().toString() + ) + } else { + client.sendTask('posthog.tasks.process_event.process_event', [], { + distinct_id, + ip, + site_url, + data, + team_id, + now, + sent_at, + }) + } } } catch (e) { Sentry.captureException(e) @@ -72,16 +87,23 @@ async function startQueueKafka( processEventBatch: (event: PluginEvent[]) => Promise<(PluginEvent | null)[]> ): Promise { const kafkaQueue = new KafkaQueue(server, processEventBatch, async (event: PluginEvent) => { - const { distinct_id, ip, site_url, team_id, now, sent_at } = event - await server.eventsProcessor.process_event_ee( - distinct_id, - ip, - site_url, - event, - team_id, - DateTime.fromISO(now), - sent_at ? DateTime.fromISO(sent_at) : null - ) + const { distinct_id, ip, site_url, team_id, now, sent_at, uuid } = event + if (!uuid) { + status.error('❓', 'UUID missing in event received from Kafka!') + return + } + if (server.PLUGIN_SERVER_INGESTION) { + await server.eventsProcessor.processEvent( + distinct_id, + ip, + site_url, + event, + team_id, + DateTime.fromISO(now), + sent_at ? DateTime.fromISO(sent_at) : null, + uuid + ) + } }) await kafkaQueue.start() diff --git a/tests/clickhouse/e2e.test.ts b/tests/clickhouse/e2e.test.ts new file mode 100644 index 00000000..78996480 --- /dev/null +++ b/tests/clickhouse/e2e.test.ts @@ -0,0 +1,63 @@ +import { LogLevel, PluginsServerConfig } from '../../src/types' +import { resetTestDatabase } from '../helpers/sql' +import { startPluginsServer } from '../../src/server' +import { makePiscina } from '../../src/worker/piscina' +import { PluginsServer } from '../../src/types' +import { createPosthog, DummyPostHog } from '../../src/extensions/posthog' +import { pluginConfig39 } from '../helpers/plugins' +import { delay, UUIDT } from '../../src/utils' +import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' +import { resetKafka } from '../helpers/kafka' +import { delayUntilEventIngested } from '../shared/process-event' +import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../../src/ingestion/topics' + +jest.setTimeout(60000) // 60 sec timeout + +const extraServerConfig: Partial = { + KAFKA_ENABLED: true, + KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', + WORKER_CONCURRENCY: 2, + PLUGIN_SERVER_INGESTION: true, + KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION, + LOG_LEVEL: LogLevel.Log, +} + +describe('e2e clickhouse ingestion', () => { + let server: PluginsServer + let stopServer: () => Promise + let posthog: DummyPostHog + + beforeAll(async () => { + await resetKafka(extraServerConfig) + }) + + beforeEach(async () => { + await resetTestDatabase(` + async function processEvent (event) { + event.properties.processed = 'hell yes' + event.properties.upperUuid = event.properties.uuid?.toUpperCase() + return event + } + `) + await resetTestDatabaseClickhouse(extraServerConfig) + const startResponse = await startPluginsServer(extraServerConfig, makePiscina) + server = startResponse.server + stopServer = startResponse.stop + posthog = createPosthog(server, pluginConfig39) + }) + + afterEach(async () => { + await stopServer() + }) + + test('event captured, processed, ingested', async () => { + expect((await server.db.fetchEvents()).length).toBe(0) + const uuid = new UUIDT().toString() + posthog.capture('custom event', { name: 'haha', uuid }) + await delayUntilEventIngested(() => server.db.fetchEvents()) + const events = await server.db.fetchEvents() + expect(events.length).toBe(1) + expect(events[0].properties.processed).toEqual('hell yes') + expect(events[0].properties.upperUuid).toEqual(uuid.toUpperCase()) + }) +}) diff --git a/tests/clickhouse/ingestion-utils.test.ts b/tests/clickhouse/ingestion-utils.test.ts new file mode 100644 index 00000000..ee2ea40f --- /dev/null +++ b/tests/clickhouse/ingestion-utils.test.ts @@ -0,0 +1,50 @@ +import { chainToElements, elementsToString } from '../../src/ingestion/utils' + +test('elementsToString and chainToElements', async () => { + const elementsString = elementsToString([ + { + tag_name: 'a', + href: '/a-url', + attr_class: ['small'], + text: 'bla bla', + attributes: { + prop: 'value', + number: 33, + 'data-attr': 'something " that; could mess up', + style: 'min-height: 100vh;', + }, + nth_child: 1, + nth_of_type: 0, + }, + { tag_name: 'button', attr_class: ['btn', 'btn-primary'], nth_child: 0, nth_of_type: 0 }, + { tag_name: 'div', nth_child: 0, nth_of_type: 0 }, + { tag_name: 'div', nth_child: 0, nth_of_type: 0, attr_id: 'nested' }, + ]) + + expect(elementsString).toEqual( + [ + 'a.small:data-attr="something \\" that; could mess up"href="/a-url"nth-child="1"nth-of-type="0"number="33"prop="value"style="min-height: 100vh;"text="bla bla"', + 'button.btn.btn-primary:nth-child="0"nth-of-type="0"', + 'div:nth-child="0"nth-of-type="0"', + 'div:attr_id="nested"nth-child="0"nth-of-type="0"', + ].join(';') + ) + + const elements = chainToElements(elementsString) + expect(elements.length).toBe(4) + expect(elements[0].tag_name).toEqual('a') + expect(elements[0].href).toEqual('/a-url') + expect(elements[0].attr_class).toEqual(['small']) + expect(elements[0].attributes).toEqual({ + prop: 'value', + number: '33', + // NB! The original Python code also does not unescape `\"` -> `"` + // Could be fixed later, but keeping as is for parity. + 'data-attr': 'something \\" that; could mess up', + style: 'min-height: 100vh;', + }) + expect(elements[0].nth_child).toEqual(1) + expect(elements[0].nth_of_type).toEqual(0) + expect(elements[1].attr_class).toEqual(['btn', 'btn-primary']) + expect(elements[3].attr_id).toEqual('nested') +}) diff --git a/tests/clickhouse/postgres-parity.test.ts b/tests/clickhouse/postgres-parity.test.ts new file mode 100644 index 00000000..a964faf1 --- /dev/null +++ b/tests/clickhouse/postgres-parity.test.ts @@ -0,0 +1,298 @@ +import { Database, LogLevel, PluginsServer, PluginsServerConfig, Team, TimestampFormat } from '../../src/types' +import { getFirstTeam, resetTestDatabase } from '../helpers/sql' +import { startPluginsServer } from '../../src/server' +import { makePiscina } from '../../src/worker/piscina' +import { createPosthog, DummyPostHog } from '../../src/extensions/posthog' +import { pluginConfig39 } from '../helpers/plugins' +import { castTimestampOrNow, UUIDT } from '../../src/utils' +import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' +import { resetKafka } from '../helpers/kafka' +import { delayUntilEventIngested } from '../shared/process-event' +import { DateTime } from 'luxon' + +jest.setTimeout(60000) // 60 sec timeout + +const extraServerConfig: Partial = { + KAFKA_ENABLED: true, + KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', + WORKER_CONCURRENCY: 2, + PLUGIN_SERVER_INGESTION: true, + LOG_LEVEL: LogLevel.Log, +} + +describe('postgres parity', () => { + let server: PluginsServer + let stopServer: () => Promise + let posthog: DummyPostHog + let team: Team + + beforeAll(async () => { + await resetKafka(extraServerConfig) + }) + + beforeEach(async () => { + await resetTestDatabase(` + async function processEvent (event) { + event.properties.processed = 'hell yes' + event.properties.upperUuid = event.properties.uuid?.toUpperCase() + return event + } + `) + await resetTestDatabaseClickhouse(extraServerConfig) + const startResponse = await startPluginsServer(extraServerConfig, makePiscina) + server = startResponse.server + stopServer = startResponse.stop + posthog = createPosthog(server, pluginConfig39) + team = await getFirstTeam(server) + }) + + afterEach(async () => { + await stopServer() + }) + + test('createPerson', async () => { + const uuid = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + true, + uuid, + ['distinct1', 'distinct2'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(person, Database.ClickHouse), 2) + + const clickHousePersons = await server.db.fetchPersons(Database.ClickHouse) + expect(clickHousePersons).toEqual([ + { + id: uuid, + created_at: expect.any(String), // '2021-02-04 00:18:26.472', + team_id: team.id, + properties: '{"userProp":"propValue"}', + is_identified: 1, + _timestamp: expect.any(String), + _offset: expect.any(Number), + }, + ]) + const clickHouseDistinctIds = await server.db.fetchDistinctIdValues(person, Database.ClickHouse) + expect(clickHouseDistinctIds).toEqual(['distinct1', 'distinct2']) + + const postgresPersons = await server.db.fetchPersons(Database.Postgres) + expect(postgresPersons).toEqual([ + { + id: expect.any(Number), + created_at: expect.any(DateTime), + properties: { + userProp: 'propValue', + }, + team_id: 2, + is_user_id: null, + is_identified: true, + uuid: uuid, + }, + ]) + const postgresDistinctIds = await server.db.fetchDistinctIdValues(person, Database.Postgres) + expect(postgresDistinctIds).toEqual(['distinct1', 'distinct2']) + + expect(person).toEqual(postgresPersons[0]) + }) + + test('updatePerson', async () => { + const uuid = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + false, + uuid, + ['distinct1', 'distinct2'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(person, Database.ClickHouse), 2) + + // update JSON and boolean to true + + await server.db.updatePerson(person, { properties: { replacedUserProp: 'propValue' }, is_identified: true }) + + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).filter((p) => p.is_identified) + ) + + const clickHousePersons = await server.db.fetchPersons(Database.ClickHouse) + const postgresPersons = await server.db.fetchPersons(Database.Postgres) + + expect(clickHousePersons.length).toEqual(1) + expect(postgresPersons.length).toEqual(1) + + expect(postgresPersons[0].is_identified).toEqual(true) + expect(postgresPersons[0].properties).toEqual({ replacedUserProp: 'propValue' }) + + expect(clickHousePersons[0].is_identified).toEqual(1) + expect(clickHousePersons[0].properties).toEqual('{"replacedUserProp":"propValue"}') + + // update date and boolean to false + + const randomDate = DateTime.utc().minus(100000).setZone('UTC') + await server.db.updatePerson(person, { created_at: randomDate, is_identified: false }) + + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).filter((p) => p.is_identified) + ) + + const clickHousePersons2 = await server.db.fetchPersons(Database.ClickHouse) + const postgresPersons2 = await server.db.fetchPersons(Database.Postgres) + + expect(clickHousePersons2.length).toEqual(1) + expect(postgresPersons2.length).toEqual(1) + + expect(postgresPersons2[0].is_identified).toEqual(false) + expect(postgresPersons2[0].created_at.toISO()).toEqual(randomDate.toISO()) + + expect(clickHousePersons2[0].is_identified).toEqual(0) + expect(clickHousePersons2[0].created_at).toEqual(castTimestampOrNow(randomDate, TimestampFormat.ClickHouse)) + }) + + test('deletePerson', async () => { + const uuid = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + false, + uuid, + ['distinct1', 'distinct2'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(person, Database.ClickHouse), 2) + + await server.db.deletePerson(person) + + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).length === 0 ? ['deleted!'] : [] + ) + + const clickHousePersons = await server.db.fetchPersons(Database.ClickHouse) + const postgresPersons = await server.db.fetchPersons(Database.Postgres) + + expect(clickHousePersons.length).toEqual(0) + expect(postgresPersons.length).toEqual(0) + + const clickHouseDistinctIdValues = await server.db.fetchDistinctIdValues(person, Database.ClickHouse) + const postgresDistinctIdValues = await server.db.fetchDistinctIdValues(person, Database.Postgres) + expect(clickHouseDistinctIdValues.length).toEqual(0) + expect(postgresDistinctIdValues.length).toEqual(0) + }) + + test('addDistinctId & moveDistinctId', async () => { + const uuid = new UUIDT().toString() + const uuid2 = new UUIDT().toString() + const person = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + true, + uuid, + ['distinct1'] + ) + const anotherPerson = await server.db.createPerson( + DateTime.utc(), + { userProp: 'propValue' }, + team.id, + null, + true, + uuid2, + ['another_distinct_id'] + ) + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse)) + const [postgresPerson] = await server.db.fetchPersons(Database.Postgres) + + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse), 1) + const clickHouseDistinctIdValues = await server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse) + const postgresDistinctIdValues = await server.db.fetchDistinctIdValues(postgresPerson, Database.Postgres) + + // check that all is in the right format + + expect(clickHouseDistinctIdValues).toEqual(['distinct1']) + expect(postgresDistinctIdValues).toEqual(['distinct1']) + + const clickHouseDistinctIds = await server.db.fetchDistinctIds(postgresPerson, Database.ClickHouse) + const postgresDistinctIds = await server.db.fetchDistinctIds(postgresPerson, Database.Postgres) + + expect(clickHouseDistinctIds).toEqual([ + { + id: expect.any(Number), + distinct_id: 'distinct1', + person_id: person.uuid, + team_id: team.id, + _timestamp: expect.any(String), + _offset: expect.any(Number), + }, + ]) + expect(postgresDistinctIds).toEqual([ + { + id: expect.any(Number), + distinct_id: 'distinct1', + person_id: person.id, + team_id: team.id, + }, + ]) + expect(clickHouseDistinctIds[0].id).toEqual(postgresDistinctIds[0].id) + + // add 'anotherOne' to person + + await server.db.addDistinctId(postgresPerson, 'anotherOne') + + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse), 2) + + const clickHouseDistinctIdValues2 = await server.db.fetchDistinctIdValues(postgresPerson, Database.ClickHouse) + const postgresDistinctIdValues2 = await server.db.fetchDistinctIdValues(postgresPerson, Database.Postgres) + + expect(clickHouseDistinctIdValues2).toEqual(['distinct1', 'anotherOne']) + expect(postgresDistinctIdValues2).toEqual(['distinct1', 'anotherOne']) + + // check anotherPerson for their initial distinct id + + const clickHouseDistinctIdValuesOther = await server.db.fetchDistinctIdValues( + anotherPerson, + Database.ClickHouse + ) + const postgresDistinctIdValuesOther = await server.db.fetchDistinctIdValues(anotherPerson, Database.Postgres) + + expect(clickHouseDistinctIdValuesOther).toEqual(['another_distinct_id']) + expect(postgresDistinctIdValuesOther).toEqual(['another_distinct_id']) + + // move 'distinct1' from person to to anotherPerson + + await server.db.moveDistinctId(postgresPerson, postgresDistinctIds[0], anotherPerson) + await delayUntilEventIngested(() => server.db.fetchDistinctIdValues(anotherPerson, Database.ClickHouse), 2) + + // it got added + + const clickHouseDistinctIdValuesMoved = await server.db.fetchDistinctIdValues( + anotherPerson, + Database.ClickHouse + ) + const postgresDistinctIdValuesMoved = await server.db.fetchDistinctIdValues(anotherPerson, Database.Postgres) + + expect(clickHouseDistinctIdValuesMoved).toEqual(['distinct1', 'another_distinct_id']) + expect(postgresDistinctIdValuesMoved).toEqual(['distinct1', 'another_distinct_id']) + + // it got removed + + const clickHouseDistinctIdValuesRemoved = await server.db.fetchDistinctIdValues( + postgresPerson, + Database.ClickHouse + ) + const postgresDistinctIdValuesRemoved = await server.db.fetchDistinctIdValues(postgresPerson, Database.Postgres) + + // The `distinct1` key is still there in clickhouse, yet ALSO there for the new person. + // Eventually this should be compacted away but it's not right now. + expect(clickHouseDistinctIdValuesRemoved).toEqual(['distinct1', 'anotherOne']) + expect(postgresDistinctIdValuesRemoved).toEqual(['anotherOne']) + }) +}) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts new file mode 100644 index 00000000..a6a9b6a6 --- /dev/null +++ b/tests/clickhouse/process-event.test.ts @@ -0,0 +1,26 @@ +import { PluginsServerConfig, Event } from '../../src/types' +import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' +import { resetKafka } from '../helpers/kafka' +import { createProcessEventTests } from '../shared/process-event' +import { KAFKA_EVENTS_PLUGIN_INGESTION } from '../../src/ingestion/topics' + +jest.setTimeout(180_000) // 3 minute timeout + +const extraServerConfig: Partial = { + KAFKA_ENABLED: true, + KAFKA_HOSTS: process.env.KAFKA_HOSTS || 'kafka:9092', + PLUGIN_SERVER_INGESTION: true, + KAFKA_CONSUMPTION_TOPIC: KAFKA_EVENTS_PLUGIN_INGESTION, +} + +describe('process event (clickhouse)', () => { + beforeAll(async () => { + await resetKafka(extraServerConfig) + }) + + beforeEach(async () => { + await resetTestDatabaseClickhouse(extraServerConfig) + }) + + createProcessEventTests('clickhouse', extraServerConfig) +}) diff --git a/tests/helpers/clickhouse.ts b/tests/helpers/clickhouse.ts new file mode 100644 index 00000000..caa09f22 --- /dev/null +++ b/tests/helpers/clickhouse.ts @@ -0,0 +1,24 @@ +import { defaultConfig } from '../../src/config' +import ClickHouse from '@posthog/clickhouse' +import { PluginsServerConfig } from '../../src/types' + +export async function resetTestDatabaseClickhouse(extraServerConfig: Partial): Promise { + const config = { ...defaultConfig, ...extraServerConfig } + const clickhouse = new ClickHouse({ + host: config.CLICKHOUSE_HOST, + port: 8123, + dataObjects: true, + queryOptions: { + database: config.CLICKHOUSE_DATABASE, + output_format_json_quote_64bit_integers: false, + }, + }) + await clickhouse.querying('TRUNCATE events') + await clickhouse.querying('TRUNCATE events_mv') + await clickhouse.querying('TRUNCATE person') + await clickhouse.querying('TRUNCATE person_distinct_id') + await clickhouse.querying('TRUNCATE person_mv') + await clickhouse.querying('TRUNCATE person_static_cohort') + await clickhouse.querying('TRUNCATE session_recording_events') + await clickhouse.querying('TRUNCATE session_recording_events_mv') +} diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts new file mode 100644 index 00000000..031ffae8 --- /dev/null +++ b/tests/helpers/kafka.ts @@ -0,0 +1,84 @@ +import { Kafka, logLevel } from 'kafkajs' +import { PluginsServerConfig } from '../../src/types' +import { delay, UUIDT } from '../../src/utils' +import { defaultConfig, overrideWithEnv } from '../../src/config' +import { + KAFKA_EVENTS, + KAFKA_EVENTS_PLUGIN_INGESTION, + KAFKA_EVENTS_WAL, + KAFKA_PERSON, + KAFKA_PERSON_UNIQUE_ID, + KAFKA_SESSION_RECORDING_EVENTS, +} from '../../src/ingestion/topics' + +/** Clear the kafka queue */ +export async function resetKafka(extraServerConfig: Partial, delayMs = 2000) { + console.log('Resetting Kafka!') + const config = { ...overrideWithEnv(defaultConfig, process.env), ...extraServerConfig } + const kafka = new Kafka({ + clientId: `plugin-server-test-${new UUIDT()}`, + brokers: (config.KAFKA_HOSTS || '').split(','), + logLevel: logLevel.WARN, + }) + const producer = kafka.producer() + const consumer = kafka.consumer({ + groupId: 'clickhouse-ingestion-test', + }) + const messages = [] + + await createTopics(kafka, [ + KAFKA_EVENTS, + KAFKA_EVENTS_PLUGIN_INGESTION, + KAFKA_EVENTS_WAL, + KAFKA_SESSION_RECORDING_EVENTS, + KAFKA_PERSON, + KAFKA_PERSON_UNIQUE_ID, + ]) + + const connected = await new Promise(async (resolve, reject) => { + console.info('setting group join and crash listeners') + const { CONNECT, GROUP_JOIN, CRASH } = consumer.events + consumer.on(CONNECT, () => { + console.log('consumer connected to kafka') + }) + consumer.on(GROUP_JOIN, () => { + console.log('joined group') + resolve() + }) + consumer.on(CRASH, ({ payload: { error } }) => reject(error)) + console.info('connecting producer') + await producer.connect() + console.info('subscribing consumer') + + await consumer.subscribe({ topic: KAFKA_EVENTS_PLUGIN_INGESTION }) + console.info('running consumer') + await consumer.run({ + eachMessage: async (payload) => { + console.info('message received!') + messages.push(payload) + }, + }) + }) + + console.info(`awaiting ${delayMs} ms before disconnecting`) + await delay(delayMs) + + console.info('disconnecting producer') + await producer.disconnect() + console.info('stopping consumer') + await consumer.stop() + console.info('disconnecting consumer') + await consumer.disconnect() + + return true +} + +async function createTopics(kafka: Kafka, topics: string[]) { + const admin = kafka.admin() + await admin.connect() + await admin.createTopics({ + waitForLeaders: true, + topics: topics.map((topic) => ({ topic })), + }) + await admin.disconnect() +} diff --git a/tests/helpers/plugins.ts b/tests/helpers/plugins.ts index 75f89e9b..224f3840 100644 --- a/tests/helpers/plugins.ts +++ b/tests/helpers/plugins.ts @@ -4,6 +4,8 @@ import path from 'path' import os from 'os' import AdmZip from 'adm-zip' +export const commonUserId = 1001 +export const commonOrganizationMembershipId = '0177364a-fc7b-0000-511c-137090b9e4e1' export const commonOrganizationId = 'ca30f2ec-e9a4-4001-bf27-3ef194086068' export const plugin60: Plugin = { diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 94eaa75d..35464c0e 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -1,48 +1,34 @@ -import { makePluginObjects, commonOrganizationId } from './plugins' +import { makePluginObjects, commonOrganizationId, commonUserId, commonOrganizationMembershipId } from './plugins' import { defaultConfig } from '../../src/config' import { Pool } from 'pg' import { delay, UUIDT } from '../../src/utils' +import { PluginsServer, PluginsServerConfig, Team } from '../../src/types' -export async function resetTestDatabase(code: string): Promise { - const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) +export async function resetTestDatabase( + code: string, + extraServerConfig: Partial = {} +): Promise { + const config = { ...defaultConfig, ...extraServerConfig } + const db = new Pool({ connectionString: config.DATABASE_URL }) const mocks = makePluginObjects(code) + await db.query('DELETE FROM posthog_element') + await db.query('DELETE FROM posthog_elementgroup') + await db.query('DELETE FROM posthog_sessionrecordingevent') + await db.query('DELETE FROM posthog_persondistinctid') + await db.query('DELETE FROM posthog_person') + await db.query('DELETE FROM posthog_event') await db.query('DELETE FROM posthog_pluginstorage') await db.query('DELETE FROM posthog_pluginattachment') await db.query('DELETE FROM posthog_pluginconfig') await db.query('DELETE FROM posthog_plugin') await db.query('DELETE FROM posthog_team') + await db.query('DELETE FROM posthog_organizationmembership') await db.query('DELETE FROM posthog_organization') + await db.query('DELETE FROM posthog_user') const teamIds = mocks.pluginConfigRows.map((c) => c.team_id) - await insertRow(db, 'posthog_organization', { - id: commonOrganizationId, - name: 'TEST ORG', - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - }) - for (const teamId of teamIds) { - await insertRow(db, 'posthog_team', { - id: teamId, - organization_id: commonOrganizationId, - app_urls: [], - name: 'TEST PROJECT', - event_names: [], - event_names_with_usage: [], - event_properties: [], - event_properties_with_usage: [], - event_properties_numerical: [], - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - anonymize_ips: false, - completed_snippet_onboarding: true, - ingested_event: true, - uuid: new UUIDT().toString(), - session_recording_opt_in: true, - plugins_opt_in: true, - opt_out_capture: false, - is_demo: false, - }) - } + await createUserTeamAndOrganization(db, teamIds[0]) + for (const plugin of mocks.pluginRows) { await insertRow(db, 'posthog_plugin', plugin) } @@ -70,3 +56,68 @@ async function insertRow(db: Pool, table: string, object: Record): throw error } } + +export async function createUserTeamAndOrganization( + db: Pool, + teamId: number, + userId: number = commonUserId, + organizationId: string = commonOrganizationId, + organizationMembershipId: string = commonOrganizationMembershipId +): Promise { + await insertRow(db, 'posthog_user', { + id: userId, + password: 'gibberish', + first_name: 'PluginTest', + last_name: 'User', + email: `test${userId}@posthog.com`, + distinct_id: `plugin_test_user_distinct_id_${userId}`, + is_staff: false, + is_active: false, + date_joined: new Date().toISOString(), + }) + await insertRow(db, 'posthog_organization', { + id: organizationId, + name: 'TEST ORG', + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + personalization: '{}', + setup_section_2_completed: true, + }) + await insertRow(db, 'posthog_organizationmembership', { + id: organizationMembershipId, + organization_id: organizationId, + user_id: userId, + level: 15, + joined_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }) + await insertRow(db, 'posthog_team', { + id: teamId, + organization_id: organizationId, + app_urls: [], + name: 'TEST PROJECT', + event_names: JSON.stringify([]), + event_names_with_usage: JSON.stringify([]), + event_properties: JSON.stringify([]), + event_properties_with_usage: JSON.stringify([]), + event_properties_numerical: JSON.stringify([]), + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + anonymize_ips: false, + completed_snippet_onboarding: true, + ingested_event: true, + uuid: new UUIDT().toString(), + session_recording_opt_in: true, + plugins_opt_in: true, + opt_out_capture: false, + is_demo: false, + }) +} + +export async function getTeams(server: PluginsServer): Promise { + return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows +} + +export async function getFirstTeam(server: PluginsServer): Promise { + return (await getTeams(server))[0] +} diff --git a/tests/postgres/e2e.test.ts b/tests/postgres/e2e.test.ts new file mode 100644 index 00000000..dc3a2b18 --- /dev/null +++ b/tests/postgres/e2e.test.ts @@ -0,0 +1,60 @@ +import { LogLevel } from '../../src/types' +import { resetTestDatabase } from '../helpers/sql' +import { startPluginsServer } from '../../src/server' +import { makePiscina } from '../../src/worker/piscina' +import { PluginsServer } from '../../src/types' +import { createPosthog, DummyPostHog } from '../../src/extensions/posthog' +import { pluginConfig39 } from '../helpers/plugins' +import { delay, UUIDT } from '../../src/utils' +import { delayUntilEventIngested } from '../shared/process-event' + +jest.setTimeout(60000) // 60 sec timeout + +describe('e2e postgres ingestion', () => { + let server: PluginsServer + let stopServer: () => Promise + let posthog: DummyPostHog + + beforeEach(async () => { + await resetTestDatabase(` + async function processEvent (event) { + event.properties.processed = 'hell yes' + event.properties.upperUuid = event.properties.uuid?.toUpperCase() + return event + } + `) + const startResponse = await startPluginsServer( + { + WORKER_CONCURRENCY: 2, + PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', + CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', + PLUGIN_SERVER_INGESTION: true, + LOG_LEVEL: LogLevel.Log, + KAFKA_ENABLED: false, + }, + makePiscina + ) + server = startResponse.server + stopServer = startResponse.stop + + await server.redis.del(server.PLUGINS_CELERY_QUEUE) + await server.redis.del(server.CELERY_DEFAULT_QUEUE) + + posthog = createPosthog(server, pluginConfig39) + }) + + afterEach(async () => { + await stopServer() + }) + + test('event captured, processed, ingested', async () => { + expect((await server.db.fetchEvents()).length).toBe(0) + const uuid = new UUIDT().toString() + posthog.capture('custom event', { name: 'haha', uuid, randomProperty: 'lololo' }) + await delayUntilEventIngested(() => server.db.fetchEvents()) + const events = await server.db.fetchEvents() + expect(events.length).toBe(1) + expect(events[0].properties.processed).toEqual('hell yes') + expect(events[0].properties.upperUuid).toEqual(uuid.toUpperCase()) + }) +}) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts new file mode 100644 index 00000000..3288aea1 --- /dev/null +++ b/tests/postgres/process-event.test.ts @@ -0,0 +1,56 @@ +import { createProcessEventTests } from '../shared/process-event' +import { createUserTeamAndOrganization } from '../helpers/sql' +import { Team } from '../../src/types' + +jest.setTimeout(600000) // 600 sec timeout + +describe('process event (postgresql)', () => { + createProcessEventTests('postgresql', {}, (response) => { + test('element group', async () => { + const { server } = response + const elements = [{ tag_name: 'button', text: 'Sign up!' }, { tag_name: 'div' }] + + const elementsHash = server!.db.createElementGroup(elements, 2) + const elementGroup = await server!.db.fetchElements() + + console.log(elementGroup) + + expect(elementGroup[0].tag_name).toEqual('button') + expect(elementGroup[1].tag_name).toEqual('div') + expect(elementGroup.length).toEqual(2) + + const elements2 = [ + { tag_name: 'button', text: 'Sign up!' }, + // make sure we remove events if we can + { tag_name: 'div', event: { id: 'blabla' } }, + ] + + const elementsHash2 = server!.db.createElementGroup(elements2, 2) + const elementGroup2 = await server!.db.fetchElements() + // we are fetching all the elements, so expect there to be no new ones + expect(elementGroup2.length).toEqual(2) + expect(elementsHash).toEqual(elementsHash2) + + await createUserTeamAndOrganization( + server!.postgres, + 3, + 1002, + '01774e2f-0d01-0000-ee94-9a238640c6ee', + '0174f81e-36f5-0000-7ef8-cc26c1fbab1c' + ) + + const teams = (await server!.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows as Team[] + + // # Test no team leakage + const team2 = teams[1] + + const elementsHash3 = server!.db.createElementGroup(elements2, 3) + const elementGroup3 = await server!.db.fetchElements() + console.log(elementGroup3) + // created new elements as it's different team even if the hash is the same + expect(elementGroup3.length).toEqual(4) + expect(elementsHash).toEqual(elementsHash2) + expect(elementsHash).toEqual(elementsHash3) + }) + }) +}) diff --git a/tests/queue.test.ts b/tests/postgres/queue.test.ts similarity index 97% rename from tests/queue.test.ts rename to tests/postgres/queue.test.ts index 78b003d8..16492522 100644 --- a/tests/queue.test.ts +++ b/tests/postgres/queue.test.ts @@ -1,9 +1,9 @@ -import { redisFactory } from './helpers/redis' -import { startQueue } from '../src/worker/queue' -import { createServer } from '../src/server' -import { LogLevel, PluginsServer } from '../src/types' -import Client from '../src/celery/client' -import { runPlugins } from '../src/plugins' +import { redisFactory } from '../helpers/redis' +import { startQueue } from '../../src/worker/queue' +import { createServer } from '../../src/server' +import { LogLevel, PluginsServer } from '../../src/types' +import Client from '../../src/celery/client' +import { runPlugins } from '../../src/plugins' jest.mock('ioredis', () => redisFactory()) diff --git a/tests/vm.test.ts b/tests/postgres/vm.test.ts similarity index 96% rename from tests/vm.test.ts rename to tests/postgres/vm.test.ts index a7a681a8..dfa6ff41 100644 --- a/tests/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -1,14 +1,14 @@ -import { createPluginConfigVM } from '../src/vm' -import { PluginConfig, PluginsServer, Plugin } from '../src/types' +import { createPluginConfigVM } from '../../src/vm' +import { PluginsServer } from '../../src/types' import { PluginEvent } from '@posthog/plugin-scaffold' -import { createServer } from '../src/server' +import { createServer } from '../../src/server' import * as fetch from 'node-fetch' -import { delay } from '../src/utils' -import Client from '../src/celery/client' -import { resetTestDatabase } from './helpers/sql' -import { pluginConfig39 } from './helpers/plugins' +import { delay } from '../../src/utils' +import Client from '../../src/celery/client' +import { resetTestDatabase } from '../helpers/sql' +import { pluginConfig39 } from '../helpers/plugins' -jest.mock('../src/celery/client') +jest.mock('../../src/celery/client') const defaultEvent = { distinct_id: 'my_id', @@ -28,7 +28,7 @@ beforeEach(async () => { afterEach(async () => { mockServer.redis.disconnect() - await mockServer.db.end() + await mockServer.postgres.end() jest.clearAllMocks() }) @@ -712,10 +712,11 @@ test('posthog in runEvery', async () => { const response = await vm.tasks.runEveryMinute.exec() expect(response).toBe('haha') - expect(Client).toHaveBeenCalledTimes(1) - expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) + expect(Client).toHaveBeenCalledTimes(2) + expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue - const mockClientInstance = (Client as any).mock.instances[0] + const mockClientInstance = (Client as any).mock.instances[1] const mockSendTask = mockClientInstance.sendTask expect(mockSendTask.mock.calls[0][0]).toEqual('posthog.tasks.process_event.process_event_with_plugins') @@ -750,10 +751,11 @@ test('posthog in runEvery with timestamp', async () => { const response = await vm.tasks.runEveryMinute.exec() expect(response).toBe('haha') - expect(Client).toHaveBeenCalledTimes(1) - expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) + expect(Client).toHaveBeenCalledTimes(2) + expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue - const mockClientInstance = (Client as any).mock.instances[0] + const mockClientInstance = (Client as any).mock.instances[1] const mockSendTask = mockClientInstance.sendTask expect(mockSendTask.mock.calls[0][0]).toEqual('posthog.tasks.process_event.process_event_with_plugins') diff --git a/tests/worker.test.ts b/tests/postgres/worker.test.ts similarity index 94% rename from tests/worker.test.ts rename to tests/postgres/worker.test.ts index 898808ee..06355dfa 100644 --- a/tests/worker.test.ts +++ b/tests/postgres/worker.test.ts @@ -1,13 +1,13 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' -import { setupPiscina } from './helpers/worker' -import { delay } from '../src/utils' -import { startPluginsServer } from '../src/server' -import { LogLevel } from '../src/types' -import { makePiscina } from '../src/worker/piscina' -import Client from '../src/celery/client' -import { resetTestDatabase } from './helpers/sql' - -jest.mock('../src/sql') +import { setupPiscina } from '../helpers/worker' +import { delay } from '../../src/utils' +import { startPluginsServer } from '../../src/server' +import { LogLevel } from '../../src/types' +import { makePiscina } from '../../src/worker/piscina' +import Client from '../../src/celery/client' +import { resetTestDatabase } from '../helpers/sql' + +jest.mock('../../src/sql') jest.setTimeout(600000) // 600 sec timeout function createEvent(index = 0): PluginEvent { @@ -31,7 +31,7 @@ test('piscina worker test', async () => { } async function runEveryDay (meta) { return 4 - } + } ` await resetTestDatabase(testCode) const piscina = setupPiscina(workerThreads, 10) diff --git a/tests/shared/process-event.ts b/tests/shared/process-event.ts new file mode 100644 index 00000000..b5e8903e --- /dev/null +++ b/tests/shared/process-event.ts @@ -0,0 +1,1173 @@ +import { PluginEvent } from '@posthog/plugin-scaffold/src/types' +import { createServer } from '../../src/server' +import { + Database, + Event, + LogLevel, + Person, + PluginsServer, + PluginsServerConfig, + SessionRecordingEvent, + Team, +} from '../../src/types' +import { createUserTeamAndOrganization, getFirstTeam, getTeams, resetTestDatabase } from '../helpers/sql' +import { EventsProcessor } from '../../src/ingestion/process-event' +import { DateTime } from 'luxon' +import { delay, UUIDT } from '../../src/utils' +import { IEvent } from '../../src/idl/protos' +import { hashElements } from '../../src/ingestion/utils' + +jest.setTimeout(600000) // 600 sec timeout + +export async function delayUntilEventIngested(fetchEvents: () => Promise, minCount = 1): Promise { + for (let i = 0; i < 30; i++) { + if ((await fetchEvents()).length >= minCount) { + return + } + await delay(500) + } +} + +async function createPerson( + server: PluginsServer, + team: Team, + distinctIds: string[], + properties: Record = {} +): Promise { + return server.db.createPerson(DateTime.utc(), properties, team.id, null, false, new UUIDT().toString(), distinctIds) +} + +type ReturnWithServer = { server?: PluginsServer; stopServer?: () => Promise } + +export const createProcessEventTests = ( + database: 'postgresql' | 'clickhouse', + extraServerConfig?: Partial, + createTests?: (response: ReturnWithServer) => void +): ReturnWithServer => { + let queryCounter = 0 + let processEventCounter = 0 + let team: Team + let server: PluginsServer + let stopServer: () => Promise + let eventsProcessor: EventsProcessor + let now = DateTime.utc() + const returned: ReturnWithServer = {} + + async function getServer(): Promise<[PluginsServer, () => Promise]> { + const [server, stopServer] = await createServer({ + PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', + CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', + LOG_LEVEL: LogLevel.Log, + ...(extraServerConfig ?? {}), + }) + + await server.redis.del(server.PLUGINS_CELERY_QUEUE) + await server.redis.del(server.CELERY_DEFAULT_QUEUE) + + const query = server.postgres.query.bind(server.postgres) + server.postgres.query = (queryText: any, values?: any, callback?: any): any => { + queryCounter++ + return query(queryText, values, callback) + } + + return [server, stopServer] + } + + async function processEvent( + distinctId: string, + ip: string, + siteUrl: string, + data: PluginEvent, + teamId: number, + now: DateTime, + sentAt: DateTime | null, + eventUuid: string + ): Promise { + const response = await eventsProcessor.processEvent( + distinctId, + ip, + siteUrl, + data, + teamId, + now, + sentAt, + eventUuid + ) + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchEvents(), ++processEventCounter) + } + return response + } + + beforeEach(async () => { + const testCode = ` + function processEvent (event, meta) { + event.properties["somewhere"] = "over the rainbow"; + return event + } + ` + await resetTestDatabase(testCode, extraServerConfig) + ;[server, stopServer] = await getServer() + returned.server = server + returned.stopServer = stopServer + eventsProcessor = new EventsProcessor(server) + queryCounter = 0 + processEventCounter = 0 + team = await getFirstTeam(server) + now = DateTime.utc() + }) + + afterEach(async () => { + await stopServer?.() + }) + + createTests?.(returned) + + test('merge people', async () => { + const p0 = await createPerson(server, team, ['person_0'], { $os: 'Microsoft' }) + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse), 1) + } + + await server.db.updatePerson(p0, { created_at: DateTime.fromISO('2020-01-01T00:00:00Z') }) + + const p1 = await createPerson(server, team, ['person_1'], { $os: 'Chrome' }) + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse), 2) + } + await server.db.updatePerson(p1, { created_at: DateTime.fromISO('2019-07-01T00:00:00Z') }) + + await processEvent( + 'person_1', + '', + '', + ({ + event: 'user signed up', + properties: {}, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + await createPerson(server, team, ['person_2'], { $os: 'Apple', $browser: 'MS Edge' }) + await createPerson(server, team, ['person_3'], { $os: 'PlayStation' }) + + if (database === 'clickhouse') { + await delayUntilEventIngested(() => server.db.fetchPersons(Database.ClickHouse), 4) + expect((await server.db.fetchPersons(Database.ClickHouse)).length).toEqual(4) + } + + expect((await server.db.fetchPersons()).length).toEqual(4) + const [person0, person1, person2, person3] = await server.db.fetchPersons() + + await eventsProcessor.mergePeople(person0, [person1, person2, person3]) + + if (database === 'clickhouse') { + await delayUntilEventIngested(async () => + (await server.db.fetchPersons(Database.ClickHouse)).length === 1 ? [1] : [] + ) + expect((await server.db.fetchPersons(Database.ClickHouse)).length).toEqual(1) + } + + expect((await server.db.fetchPersons()).length).toEqual(1) + + const [person] = await server.db.fetchPersons() + + expect(person.properties).toEqual({ $os: 'Microsoft', $browser: 'MS Edge' }) + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['person_0', 'person_1', 'person_2', 'person_3']) + expect(person.created_at.toISO()).toEqual(DateTime.fromISO('2019-07-01T00:00:00Z').setZone('UTC').toISO()) + }) + + test('capture new person', async () => { + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [true, team.id]) + team = await getFirstTeam(server) + + expect(team.event_names).toEqual([]) + + await processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + if (database === 'clickhouse') { + expect(queryCounter).toBe(8) + } else if (database === 'postgresql') { + expect(queryCounter).toBe(12) + } + + // capture a second time to verify e.g. event_names is not ['$autocapture', '$autocapture'] + await processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const events = await server.db.fetchEvents() + const persons = await server.db.fetchPersons() + expect(events.length).toEqual(2) + expect(persons.length).toEqual(1) + + const [person] = persons + const distinctIds = await server.db.fetchDistinctIdValues(person) + + const [event] = events as Event[] + expect(event.distinct_id).toEqual('2') + expect(distinctIds).toEqual(['2']) + expect(event.event).toEqual('$autocapture') + + const elements = await server.db.fetchElements(event) + expect(elements[0].tag_name).toEqual('a') + expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) + expect(elements[1].order).toEqual(1) + expect(elements[1].text).toEqual('💻') + + if (database === 'clickhouse') { + expect(hashElements(elements)).toEqual('0679137c0cd2408a2906839143e7a71f') + } else if (database === 'postgresql') { + expect(event.elements_hash).toEqual('0679137c0cd2408a2906839143e7a71f') + } + + team = await getFirstTeam(server) + expect(team.event_names).toEqual(['$autocapture']) + expect(team.event_names_with_usage).toEqual([{ event: '$autocapture', volume: null, usage_count: null }]) + expect(team.event_properties).toEqual(['distinct_id', 'token', '$ip']) + expect(team.event_properties_with_usage).toEqual([ + { key: 'distinct_id', usage_count: null, volume: null }, + { key: 'token', usage_count: null, volume: null }, + { key: '$ip', usage_count: null, volume: null }, + ]) + }) + + test('capture no element', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await processEvent( + 'asdfasdfasdf', + '', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual(['asdfasdfasdf']) + const [event] = await server.db.fetchEvents() + expect(event.event).toBe('$pageview') + }) + + test('capture sent_at', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }) + + await processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await server.db.fetchEvents() + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture sent_at no timezones', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }).setZone('UTC+4') + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }).setZone('UTC+4') + + // TODO: not sure if this is correct? + // tomorrow = tomorrow.replace(tzinfo=None) + // tomorrow_sent_at = tomorrow_sent_at.replace(tzinfo=None) + + await processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow, + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await server.db.fetchEvents() + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture no sent_at', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + + await processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + null, + new UUIDT().toString() + ) + + const [event] = await server.db.fetchEvents() + const difference = tomorrow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + expect(difference).toBeLessThan(1) + }) + + test('ip capture', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await server.db.fetchEvents() + expect(event.properties['$ip']).toBe('11.12.13.14') + }) + + test('ip override', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { $ip: '1.0.0.1', distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [event] = await server.db.fetchEvents() + expect(event.properties['$ip']).toBe('1.0.0.1') + }) + + test('anonymized ip capture', async () => { + await server.db.postgresQuery('update posthog_team set anonymize_ips = $1', [true]) + await createPerson(server, team, ['asdfasdfasdf']) + + await processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [event] = await server.db.fetchEvents() + expect(event.properties['$ip']).not.toBeDefined() + }) + + test('alias', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('alias reverse', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await processEvent( + 'old_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'old_distinct_id', token: team.api_token, alias: 'new_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('alias twice', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchPersons()).length).toBe(1) + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + + await createPerson(server, team, ['old_distinct_id_2']) + expect((await server.db.fetchPersons()).length).toBe(2) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id_2' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await server.db.fetchEvents()).length).toBe(2) + expect((await server.db.fetchPersons()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + 'old_distinct_id_2', + ]) + }) + + test('alias before person', async () => { + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + expect((await server.db.fetchPersons()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ + 'new_distinct_id', + 'old_distinct_id', + ]) + }) + + test('alias both existing', async () => { + await createPerson(server, team, ['old_distinct_id']) + await createPerson(server, team, ['new_distinct_id']) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + expect(await server.db.fetchDistinctIdValues((await server.db.fetchPersons())[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('offset timestamp', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await server.db.fetchEvents()).length).toBe(1) + + const [event] = await server.db.fetchEvents() + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') + }) + + test('offset timestamp no sent_at', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + null, + new UUIDT().toString() + ) + expect((await server.db.fetchEvents()).length).toBe(1) + + const [event] = await server.db.fetchEvents() + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') + }) + + test('alias merge properties', async () => { + await createPerson(server, team, ['old_distinct_id'], { + key_on_both: 'old value both', + key_on_old: 'old value', + }) + await createPerson(server, team, ['new_distinct_id'], { + key_on_both: 'new value both', + key_on_new: 'new value', + }) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + expect((await server.db.fetchPersons()).length).toBe(1) + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['old_distinct_id', 'new_distinct_id']) + expect(person.properties).toEqual({ + key_on_both: 'new value both', + key_on_new: 'new value', + key_on_old: 'old value', + }) + }) + + test('long htext', async () => { + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 'new_distinct_id', + token: team.api_token, + $elements: [ + { + tag_name: 'a', + $el_text: 'a'.repeat(2050), + attr__href: 'a'.repeat(2050), + nth_child: 1, + nth_of_type: 2, + attr__class: 'btn btn-sm', + }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [event] = (await server.db.fetchEvents()) as Event[] + const [element] = await server.db.fetchElements(event) + expect(element.href?.length).toEqual(2048) + expect(element.text?.length).toEqual(400) + if (database === 'postgresql') { + expect(event.elements_hash).toEqual('c2659b28e72835706835764cf7f63c2a') + } else if (database === 'clickhouse') { + expect(hashElements([element])).toEqual('c2659b28e72835706835764cf7f63c2a') + } + }) + + test('capture first team event', async () => { + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [false, team.id]) + + eventsProcessor.posthog = { + identify: jest.fn((distinctId) => true), + capture: jest.fn((event, properties) => true), + } as any + + await processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 1, + token: team.api_token, + $elements: [{ tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(eventsProcessor.posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id_1001') + expect(eventsProcessor.posthog.capture).toHaveBeenCalledWith('first team event ingested', { + team: team.uuid, + }) + + team = await getFirstTeam(server) + expect(team.ingested_event).toEqual(true) + + const [event] = (await server.db.fetchEvents()) as Event[] + if (database === 'postgresql') { + expect(event.elements_hash).toEqual('a89021a60b3497d24e93ae181fba01aa') + } else if (database === 'clickhouse') { + const elements = await server.db.fetchElements(event) + expect(hashElements(elements)).toEqual('a89021a60b3497d24e93ae181fba01aa') + } + }) + + test('snapshot event stored as session_recording_event', async () => { + await eventsProcessor.processEvent( + 'some-id', + '', + '', + ({ + event: '$snapshot', + properties: { $session_id: 'abcf-efg', $snapshot_data: { timestamp: 123 } }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + await delayUntilEventIngested(() => server.db.fetchSessionRecordingEvents()) + + const events = await server.db.fetchEvents() + expect(events.length).toEqual(0) + + const sessionRecordingEvents = await server.db.fetchSessionRecordingEvents() + expect(sessionRecordingEvents.length).toBe(1) + + const [event] = sessionRecordingEvents + expect(event.session_id).toEqual('abcf-efg') + expect(event.distinct_id).toEqual('some-id') + expect(event.snapshot_data).toEqual({ timestamp: 123 }) + }) + + test('identify set', async () => { + await createPerson(server, team, ['distinct_id']) + + await processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + + const [event] = await server.db.fetchEvents() + expect(event.properties['$set']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await server.db.fetchEvents()).length).toBe(2) + const [person2] = await server.db.fetchPersons() + expect(person2.properties).toEqual({ a_prop: 'test-2', b_prop: 'test-2b', c_prop: 'test-1' }) + }) + + test('identify set_once', async () => { + await createPerson(server, team, ['distinct_id']) + + await processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + + const [event] = await server.db.fetchEvents() + expect(event.properties['$set_once']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await server.db.fetchEvents()).length).toBe(2) + const [person2] = await server.db.fetchPersons() + expect(person2.properties).toEqual({ a_prop: 'test-1', b_prop: 'test-2b', c_prop: 'test-1' }) + }) + + test('distinct with anonymous_id', async () => { + await createPerson(server, team, ['anonymous_id']) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await server.db.fetchEvents()).length).toBe(1) + const [event] = await server.db.fetchEvents() + expect(event.properties['$set']).toEqual({ a_prop: 'test' }) + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test' }) + + // check no errors as this call can happen multiple times + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + }) + + // This case is likely to happen after signup, for example: + // 1. User browses website with anonymous_id + // 2. User signs up, triggers event with their new_distinct_id (creating a new Person) + // 3. In the frontend, try to alias anonymous_id with new_distinct_id + // Result should be that we end up with one Person with both ID's + test('distinct with anonymous_id which was already created', async () => { + await createPerson(server, team, ['anonymous_id']) + await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person] = await server.db.fetchPersons() + expect(await server.db.fetchDistinctIdValues(person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties['email']).toEqual('someone@gmail.com') + }) + + test('distinct with multiple anonymous_ids which were already created', async () => { + await createPerson(server, team, ['anonymous_id']) + await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const persons1 = await server.db.fetchPersons() + expect(persons1.length).toBe(1) + expect(await server.db.fetchDistinctIdValues(persons1[0])).toEqual(['anonymous_id', 'new_distinct_id']) + expect(persons1[0].properties['email']).toEqual('someone@gmail.com') + + await createPerson(server, team, ['anonymous_id_2']) + + await processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id_2', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const persons2 = await server.db.fetchPersons() + expect(persons2.length).toBe(1) + expect(await server.db.fetchDistinctIdValues(persons2[0])).toEqual([ + 'anonymous_id', + 'new_distinct_id', + 'anonymous_id_2', + ]) + expect(persons2[0].properties['email']).toEqual('someone@gmail.com') + }) + + test('distinct team leakage', async () => { + await createUserTeamAndOrganization( + server.postgres, + 3, + 1002, + '01774e2f-0d01-0000-ee94-9a238640c6ee', + '0174f81e-36f5-0000-7ef8-cc26c1fbab1c' + ) + const team2 = (await getTeams(server))[1] + await createPerson(server, team2, ['2'], { email: 'team2@gmail.com' }) + await createPerson(server, team, ['1', '2']) + + await processEvent( + '2', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: '1', + token: team.api_token, + distinct_id: '2', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const people = await server.db.fetchPersons() + expect(people.length).toEqual(2) + expect(people[1].team_id).toEqual(team.id) + expect(people[1].properties).toEqual({}) + expect(await server.db.fetchDistinctIdValues(people[1])).toEqual(['1', '2']) + expect(people[0].team_id).toEqual(team2.id) + expect(await server.db.fetchDistinctIdValues(people[0])).toEqual(['2']) + }) + + test('set is_identified', async () => { + const distinct_id = '777' + const person1 = await createPerson(server, team, [distinct_id]) + expect(person1.is_identified).toBe(false) + + await processEvent( + distinct_id, + '', + '', + ({ event: '$identify', properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person2] = await server.db.fetchPersons() + expect(person2.is_identified).toBe(true) + }) + + test('team event_properties', async () => { + expect(team.event_properties_numerical).toEqual([]) + + await processEvent( + 'xxx', + '', + '', + ({ event: 'purchase', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + team = await getFirstTeam(server) + expect(team.event_properties).toEqual(['price', 'name', '$ip']) + expect(team.event_properties_numerical).toEqual(['price']) + }) + + test('event name object json', async () => { + await processEvent( + 'xxx', + '', + '', + ({ event: { 'event name': 'as object' }, properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await server.db.fetchEvents() + expect(event.event).toEqual('{"event name":"as object"}') + }) + + test('event name array json', async () => { + await processEvent( + 'xxx', + '', + '', + ({ event: ['event name', 'a list'], properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await server.db.fetchEvents() + expect(event.event).toEqual('["event name","a list"]') + }) + + test('long event name substr', async () => { + await processEvent( + 'xxx', + '', + '', + ({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + + const [event] = await server.db.fetchEvents() + expect(event.event?.length).toBe(200) + }) + + test('throws with bad uuid', async () => { + await expect( + processEvent( + 'xxx', + '', + '', + ({ event: 'E', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + 'this is not an uuid' + ) + ).rejects.toEqual(new Error('Not a valid UUID: "this is not an uuid"')) + + await expect( + processEvent( + 'xxx', + '', + '', + ({ event: 'E', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + null as any + ) + ).rejects.toEqual(new Error('Not a valid UUID: "null"')) + }) + + return returned +} diff --git a/tests/sql.test.ts b/tests/sql.test.ts index 6315c4ce..5708013d 100644 --- a/tests/sql.test.ts +++ b/tests/sql.test.ts @@ -28,7 +28,7 @@ test('getPluginAttachmentRows', async () => { team_id: 2, }, ]) - server.db.query("update posthog_team set plugins_opt_in='f'") + server.db.postgresQuery("update posthog_team set plugins_opt_in='f'") const rows2 = await getPluginAttachmentRows(server) expect(rows2).toEqual([]) }) @@ -49,7 +49,7 @@ test('getPluginConfigRows', async () => { team_id: 2, }, ]) - server.db.query("update posthog_team set plugins_opt_in='f'") + server.db.postgresQuery("update posthog_team set plugins_opt_in='f'") const rows2 = await getPluginConfigRows(server) expect(rows2).toEqual([]) }) @@ -94,7 +94,7 @@ test('getPluginRows', async () => { url: 'https://www.npmjs.com/package/posthog-maxmind-plugin', }, ]) - server.db.query("update posthog_team set plugins_opt_in='f'") + server.db.postgresQuery("update posthog_team set plugins_opt_in='f'") const rows2 = await getPluginRows(server) expect(rows2).toEqual([]) }) @@ -109,17 +109,17 @@ test('setError', async () => { config: {}, error: undefined, } - server.db.query = jest.fn() as any + server.db.postgresQuery = jest.fn() as any await setError(server, null, pluginConfig39) - expect(server.db.query).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ + expect(server.db.postgresQuery).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ null, pluginConfig39.id, ]) const pluginError: PluginError = { message: 'error happened', time: 'now' } await setError(server, pluginError, pluginConfig39) - expect(server.db.query).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ + expect(server.db.postgresQuery).toHaveBeenCalledWith('UPDATE posthog_pluginconfig SET error = $1 WHERE id = $2', [ pluginError, pluginConfig39.id, ]) diff --git a/tests/utils.test.ts b/tests/utils.test.ts index f9fc5011..6cdef39c 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -7,6 +7,8 @@ import { cloneObject, UUID, UUIDT, + sanitizeSqlIdentifier, + escapeClickHouseString, } from '../src/utils' import { randomBytes } from 'crypto' import { LogLevel } from '../src/types' @@ -295,3 +297,23 @@ describe('UUIDT', () => { expect(uuidtString.slice(14, 18)).toEqual('0000') }) }) + +describe('sanitizeSqlIdentifier', () => { + it('removes all characters that are neither letter, digit or underscore and adds quotes around identifier', () => { + const rawIdentifier = 'some_field"; DROP TABLE actually_an_injection-9;' + + const sanitizedIdentifier = sanitizeSqlIdentifier(rawIdentifier) + + expect(sanitizedIdentifier).toStrictEqual('some_fieldDROPTABLEactually_an_injection9') + }) +}) + +describe('escapeClickHouseString', () => { + it('escapes single quotes and slashes', () => { + const rawString = "insert'escape \\" + + const sanitizedString = escapeClickHouseString(rawString) + + expect(sanitizedString).toStrictEqual("insert\\'escape \\\\") + }) +}) diff --git a/tsconfig.json b/tsconfig.json index c606b655..4c7ac88e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,7 +12,7 @@ "sourceMap": true, "baseUrl": "./src", "outDir": "./dist", - "types": ["node", "jest"], + "types": ["node", "jest", "long"], "resolveJsonModule": true, "strict": true }, diff --git a/yarn.lock b/yarn.lock index b7e09dd1..6e0c2e38 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1179,10 +1179,68 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" -"@posthog/plugin-scaffold@0.2.6": - version "0.2.6" - resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.6.tgz#5e27067e19ce42a78b67fbbafc001f816ccb0c70" - integrity sha512-W4Ata+n1Zsb+yAVlAQv3OEuHOmwz3Fp6en9iXq5dNNM0gxFoYtKAhzITvz7idAvIy15sa2244E8itSTbqEzzKA== +"@posthog/clickhouse@^1.7.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@posthog/clickhouse/-/clickhouse-1.7.0.tgz#21fa1e8cfa0637b688f91964e0efeedbf4cf7a3c" + integrity sha512-B8hZ8Dh2EoJoDb7Gx38ylBQM92oON/X2IxXCb7BfYStk3m17nStcAyaCsc2zbvxC0fFfTMU8lFRiFSEJmijkyg== + +"@posthog/plugin-scaffold@0.2.8": + version "0.2.8" + resolved "https://registry.yarnpkg.com/@posthog/plugin-scaffold/-/plugin-scaffold-0.2.8.tgz#14ee85afa9a91625bbae4e223ce03d215a22e8d7" + integrity sha512-RcjmiPe3fAgLR/qyqTxnTg3b9WdoOAeV4tEPvisEFnej3mLenxp7bgUY5W1lxSfYoDDLGx1zSRVnBQL56w5KTQ== + +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha1-m4sMxmPWaafY9vXQiToU00jzD78= + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha1-NVy8mLr61ZePntCV85diHx0Ga3A= + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU= + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E= + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik= + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha1-bMKyDFya1q0NzP0hynZz2Nf79o0= + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q= + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= "@sentry/core@5.29.0": version "5.29.0" @@ -1372,6 +1430,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/long@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" + integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== + "@types/luxon@^1.25.0": version "1.25.0" resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-1.25.0.tgz#3d6fe591fac874f48dd225cb5660b2b785a21a05" @@ -1397,6 +1460,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.12.tgz#0b1d86f8c40141091285dea02e4940df73bba43f" integrity sha512-ASH8OPHMNlkdjrEdmoILmzFfsJICvhBsFfAum4aKZ/9U4B6M6tTmTPh+f3ttWdD74CEGV5XvXWkbyfSdXaTd7g== +"@types/node@^13.7.0": + version "13.13.35" + resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.35.tgz#d417b48313d691f5c8ff9c52cbc19cdecd306b5e" + integrity sha512-q9aeOGwv+RRou/ca4aJVUM/jD5u7LBexu+rq9PkA/NhHNn8JifcMo94soKm0b6JGSfw/PSNdqtc428OscMvEYA== + "@types/normalize-package-data@^2.4.0": version "2.4.0" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" @@ -2627,9 +2695,9 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: safe-buffer "^5.0.1" electron-to-chromium@^1.3.621: - version "1.3.627" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.627.tgz#4acdbbbbe31eb605fba8380063fd9c8a7e5ca4a0" - integrity sha512-O5IVRS4sCxP2+vECAp7uHkaI8V+dKYpuCyBcLn+hqVAOy/RONd8zx+6eH7TuWSTBYs/oUrzBXkNMZuVsQd58kQ== + version "1.3.622" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.622.tgz#9726bd2e67a5462154750ce9701ca6af07d07877" + integrity sha512-AJT0Fm1W0uZlMVVkkJrcCVvczDuF8tPm3bwzQf5WO8AaASB2hwTRP7B8pU5rqjireH+ib6am8+hH5/QkXzzYKw== emittery@^0.7.1: version "0.7.2" @@ -3540,9 +3608,9 @@ globby@^11.0.1: slash "^3.0.0" google-auth-library@^6.1.1: - version "6.1.3" - resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.3.tgz#39d868140b70d0c4b32c6f6d8f4ccc1400d84dca" - integrity sha512-m9mwvY3GWbr7ZYEbl61isWmk+fvTmOt0YNUfPOUY2VH8K5pZlAIWJjxEi0PqR3OjMretyiQLI6GURMrPSwHQ2g== + version "6.1.4" + resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.4.tgz#bc70c4f3b6681ae5273343466bcef37577b7ee44" + integrity sha512-q0kYtGWnDd9XquwiQGAZeI2Jnglk7NDi0cChE4tWp6Kpo/kbqnt9scJb0HP+/xqt03Beqw/xQah1OPrci+pOxw== dependencies: arrify "^2.0.0" base64-js "^1.3.0" @@ -4872,6 +4940,11 @@ long-timeout@0.1.1: resolved "https://registry.yarnpkg.com/long-timeout/-/long-timeout-0.1.1.tgz#9721d788b47e0bcb5a24c2e2bee1a0da55dab514" integrity sha1-lyHXiLR+C8taJMLivuGg2lXatRQ= +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -5700,6 +5773,11 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" +posthog-js-lite@^0.0.5: + version "0.0.5" + resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.5.tgz#984a619190d1c4ef003cb81194d0a6276491d2e8" + integrity sha512-xHVZ9qbBdoqK7G1JVA4k6nheWu50aiEwjJIPQw7Zhi705aX5wr2W8zxdmvtadeEE5qbJNg+/uZ7renOQoqcbJw== + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -5752,6 +5830,25 @@ prop-types@^15.7.2: object-assign "^4.1.1" react-is "^16.8.1" +protobufjs@^6.10.2: + version "6.10.2" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.10.2.tgz#b9cb6bd8ec8f87514592ba3fdfd28e93f33a469b" + integrity sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.1" + "@types/node" "^13.7.0" + long "^4.0.0" + proxy-addr@^2.0.5: version "2.0.6" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.6.tgz#fdc2336505447d3f2f2c638ed272caf614bbb2bf"