From 49490133fa444c7fd91a455a083653319c72f6d1 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 09:29:04 +0100 Subject: [PATCH 01/47] start with the postgres event ingestion process event tests --- src/ingestion/process-event.ts | 28 ++++++-- src/types.ts | 5 ++ tests/postgres/process-event.test.ts | 97 ++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 tests/postgres/process-event.test.ts diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 5f769cca..12784086 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -343,7 +343,7 @@ export class EventsProcessor { } catch {} } - return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList) + return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList, siteUrl) } private async storeNamesAndProperties(team: Team, event: string, properties: Properties): Promise { @@ -401,7 +401,8 @@ export class EventsProcessor { distinctId: string, properties?: Properties, timestamp?: DateTime | string, - elements?: Element[] + elements?: Element[], + siteUrl?: string ): Promise { const timestampString = castTimestampOrNow(timestamp) const elementsChain = elements && elements.length ? elementsToString(elements) : '' @@ -417,10 +418,25 @@ export class EventsProcessor { createdAt: timestampString, } - await this.kafkaProducer.send({ - topic: KAFKA_EVENTS, - messages: [{ key: uuid, value: EventProto.encodeDelimited(EventProto.create(data)).finish() as Buffer }], - }) + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_EVENTS, + messages: [ + { + key: uuid, + value: EventProto.encodeDelimited(EventProto.create(data)).finish() as Buffer, + }, + ], + }) + } else { + // TODO: add element_group code! + // https://github.com/PostHog/posthog/blob/5d5ede19e4799dc71ffd5ec18e65bd969520b543/posthog/models/event.py#L235 + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_event (created_at, event, properties, team_id, site_url, timestamp, elements) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', + [data.createdAt, data.event, data.properties, data.teamId, siteUrl, data.timestamp, data.elementsChain] + ) + const eventCreated = insertResult.rows[0] as Event + } return data } diff --git a/src/types.ts b/src/types.ts index 6a49fc31..bb6f6d34 100644 --- a/src/types.ts +++ b/src/types.ts @@ -232,6 +232,11 @@ export interface Element { group_id?: number } +/** Usable Event model. */ +export interface Event { + event?: string +} + /** Properties shared by RawPerson and Person. */ export interface BasePerson { id: number diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts new file mode 100644 index 00000000..6a18062c --- /dev/null +++ b/tests/postgres/process-event.test.ts @@ -0,0 +1,97 @@ +import { PluginEvent } from '@posthog/plugin-scaffold/src/types' +import { setupPiscina } from '../helpers/worker' +import { delay } from '../../src/utils' +import { createServer, startPluginsServer } from '../../src/server' +import { LogLevel, PluginsServer, Team, Event } from '../../src/types' +import { makePiscina } from '../../src/worker/piscina' +import Client from '../../src/celery/client' +import { resetTestDatabase } from '../helpers/sql' +import { EventsProcessor } from '../../src/ingestion/process-event' +import { DateTime } from 'luxon' + +// jest.mock('../src/sql') +jest.setTimeout(600000) // 600 sec timeout + +let team: Team +let server: PluginsServer +let stopServer: () => Promise +let eventsProcessor: EventsProcessor +let now = DateTime.utc() + +function createEvent(event = {}): PluginEvent { + return { + distinct_id: 'my_id', + ip: '127.0.0.1', + site_url: 'http://localhost', + team_id: 2, + now: now?.toISO() || new Date().toISOString(), + event: 'default event', + properties: {}, + ...event, + } +} + +async function getServer(): Promise<[PluginsServer, () => Promise]> { + const [server, stopServer] = await createServer({ + PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', + CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', + LOG_LEVEL: LogLevel.Log, + }) + + await server.redis.del(server.PLUGINS_CELERY_QUEUE) + await server.redis.del(server.CELERY_DEFAULT_QUEUE) + return [server, stopServer] +} + +beforeEach(async () => { + const testCode = ` + function processEvent (event, meta) { + event.properties["somewhere"] = "over the rainbow"; + return event + } + ` + await resetTestDatabase(testCode) + ;[server, stopServer] = await getServer() + eventsProcessor = new EventsProcessor(server) + team = (await server.db.postgresQuery('SELECT * FROM posthog_team LIMIT 1')).rows[0] + now = DateTime.utc() +}) + +afterEach(async () => { + await stopServer?.() +}) + +async function getEvents(): Promise { + const insertResult = await server.db.postgresQuery('SELECT * FROM posthog_event') + return insertResult.rows as Event[] +} + +// +// def test_long_event_name_substr(self) -> None: +// process_event( +// "xxx", +// "", +// "", +// {"event": "E" * 300, "properties": {"price": 299.99, "name": "AirPods Pro"},}, +// self.team.pk, +// now().isoformat(), +// now().isoformat(), +// ) +// event = get_events()[0] +// self.assertEqual(len(event.event), 200) + +test('long event name substr', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + createEvent({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } }), + team.id, + DateTime.utc(), + DateTime.utc(), + 'uuid' + ) + + const [event] = await getEvents() + expect(event.event?.length).toBe(200) +}) From 1ab2d146ae78503dcad534db4657fd54225af5a3 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 10:53:00 +0100 Subject: [PATCH 02/47] get first test to work --- src/ingestion/process-event.ts | 14 ++++++++++++-- tests/helpers/sql.ts | 13 ++++++++----- tests/postgres/process-event.test.ts | 21 +-------------------- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 12784086..3b96f24c 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -431,9 +431,19 @@ export class EventsProcessor { } else { // TODO: add element_group code! // https://github.com/PostHog/posthog/blob/5d5ede19e4799dc71ffd5ec18e65bd969520b543/posthog/models/event.py#L235 + const elementsHash = '' const insertResult = await this.db.postgresQuery( - 'INSERT INTO posthog_event (created_at, event, properties, team_id, site_url, timestamp, elements) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', - [data.createdAt, data.event, data.properties, data.teamId, siteUrl, data.timestamp, data.elementsChain] + 'INSERT INTO posthog_event (created_at, event, distinct_id, properties, team_id, timestamp, elements, elements_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *', + [ + data.createdAt, + data.event, + distinctId, + data.properties, + data.teamId, + data.timestamp, + JSON.stringify(elements || []), + elementsHash, + ] ) const eventCreated = insertResult.rows[0] as Event } diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index f0710b0b..0a93aab4 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -6,6 +6,9 @@ import { delay, UUIDT } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) const mocks = makePluginObjects(code) + await db.query('DELETE FROM posthog_persondistinctid') + await db.query('DELETE FROM posthog_person') + await db.query('DELETE FROM posthog_event') await db.query('DELETE FROM posthog_pluginstorage') await db.query('DELETE FROM posthog_pluginattachment') await db.query('DELETE FROM posthog_pluginconfig') @@ -26,11 +29,11 @@ export async function resetTestDatabase(code: string): Promise { organization_id: commonOrganizationId, app_urls: [], name: 'TEST PROJECT', - event_names: [], - event_names_with_usage: [], - event_properties: [], - event_properties_with_usage: [], - event_properties_numerical: [], + event_names: JSON.stringify(['test']), + event_names_with_usage: JSON.stringify([]), + event_properties: JSON.stringify([]), + event_properties_with_usage: JSON.stringify([]), + event_properties_numerical: JSON.stringify([]), created_at: new Date().toISOString(), updated_at: new Date().toISOString(), anonymize_ips: false, diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 6a18062c..4776d802 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,15 +1,10 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' -import { setupPiscina } from '../helpers/worker' -import { delay } from '../../src/utils' -import { createServer, startPluginsServer } from '../../src/server' +import { createServer } from '../../src/server' import { LogLevel, PluginsServer, Team, Event } from '../../src/types' -import { makePiscina } from '../../src/worker/piscina' -import Client from '../../src/celery/client' import { resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' -// jest.mock('../src/sql') jest.setTimeout(600000) // 600 sec timeout let team: Team @@ -66,20 +61,6 @@ async function getEvents(): Promise { return insertResult.rows as Event[] } -// -// def test_long_event_name_substr(self) -> None: -// process_event( -// "xxx", -// "", -// "", -// {"event": "E" * 300, "properties": {"price": 299.99, "name": "AirPods Pro"},}, -// self.team.pk, -// now().isoformat(), -// now().isoformat(), -// ) -// event = get_events()[0] -// self.assertEqual(len(event.event), 200) - test('long event name substr', async () => { await eventsProcessor.processEvent( 'xxx', From 4e4aad90c873cbb73eb3571c2b941d76344be2e0 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 10:57:06 +0100 Subject: [PATCH 03/47] remove siteUrl --- src/ingestion/process-event.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 3b96f24c..5cac8b86 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -343,7 +343,7 @@ export class EventsProcessor { } catch {} } - return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList, siteUrl) + return await this.createEvent(eventUuid, event, team, distinctId, properties, timestamp, elementsList) } private async storeNamesAndProperties(team: Team, event: string, properties: Properties): Promise { @@ -401,8 +401,7 @@ export class EventsProcessor { distinctId: string, properties?: Properties, timestamp?: DateTime | string, - elements?: Element[], - siteUrl?: string + elements?: Element[] ): Promise { const timestampString = castTimestampOrNow(timestamp) const elementsChain = elements && elements.length ? elementsToString(elements) : '' From 4a80f46162207087396281c44d2c698262a86f74 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 10:57:23 +0100 Subject: [PATCH 04/47] pass partial event to test that it still works and retain parity with the django test --- tests/postgres/process-event.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 4776d802..d9ff8de2 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -66,7 +66,7 @@ test('long event name substr', async () => { 'xxx', '', '', - createEvent({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } }), + ({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, team.id, DateTime.utc(), DateTime.utc(), From ab5e5340e41ddc83142dce1d8ce1d254988293c2 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 11:18:11 +0100 Subject: [PATCH 05/47] another test --- src/types.ts | 8 +++++ tests/postgres/process-event.test.ts | 53 +++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/types.ts b/src/types.ts index bb6f6d34..159b0ac3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -234,7 +234,15 @@ export interface Element { /** Usable Event model. */ export interface Event { + id: number event?: string + properties: Record + elements?: Element[] + timestamp: string + team_id: number + distinct_id: string + elements_hash: string + created_at: DateTime } /** Properties shared by RawPerson and Person. */ diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index d9ff8de2..e7310985 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,9 +1,10 @@ -import { PluginEvent } from '@posthog/plugin-scaffold/src/types' +import { PluginEvent, Properties } from '@posthog/plugin-scaffold/src/types' import { createServer } from '../../src/server' -import { LogLevel, PluginsServer, Team, Event } from '../../src/types' +import { LogLevel, PluginsServer, Team, Event, Person, PersonDistinctId } from '../../src/types' import { resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' +import { UUIDT } from '../../src/utils' jest.setTimeout(600000) // 600 sec timeout @@ -57,10 +58,54 @@ afterEach(async () => { }) async function getEvents(): Promise { - const insertResult = await server.db.postgresQuery('SELECT * FROM posthog_event') - return insertResult.rows as Event[] + const result = await server.db.postgresQuery('SELECT * FROM posthog_event') + return result.rows as Event[] } +async function getPersons(): Promise { + const result = await server.db.postgresQuery('SELECT * FROM posthog_person') + return result.rows as Person[] +} + +async function getDistinctIds(person: Person) { + const result = await server.db.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2', + [person.id, person.team_id] + ) + return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) +} + +async function createPerson(team: Team, distinctIds: string[]) { + const person = await server.db.createPerson(DateTime.utc(), {}, team.id, null, false, new UUIDT().toString()) + for (const distinctId of distinctIds) { + await server.db.addDistinctId(person, distinctId) + } + + return person +} + +test('capture no element', async () => { + await createPerson(team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + + expect(await getDistinctIds((await getPersons())[0])).toEqual(['asdfasdfasdf']) + const [event] = await getEvents() + expect(event.event).toBe('$pageview') +}) + test('long event name substr', async () => { await eventsProcessor.processEvent( 'xxx', From 0811e6948888b51b7c5712c77258f0d292086d25 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 11:24:11 +0100 Subject: [PATCH 06/47] refactor --- tests/postgres/process-event.test.ts | 117 ++++++++++++--------------- 1 file changed, 53 insertions(+), 64 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index e7310985..bc57b4dd 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -14,19 +14,6 @@ let stopServer: () => Promise let eventsProcessor: EventsProcessor let now = DateTime.utc() -function createEvent(event = {}): PluginEvent { - return { - distinct_id: 'my_id', - ip: '127.0.0.1', - site_url: 'http://localhost', - team_id: 2, - now: now?.toISO() || new Date().toISOString(), - event: 'default event', - properties: {}, - ...event, - } -} - async function getServer(): Promise<[PluginsServer, () => Promise]> { const [server, stopServer] = await createServer({ PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', @@ -39,24 +26,6 @@ async function getServer(): Promise<[PluginsServer, () => Promise]> { return [server, stopServer] } -beforeEach(async () => { - const testCode = ` - function processEvent (event, meta) { - event.properties["somewhere"] = "over the rainbow"; - return event - } - ` - await resetTestDatabase(testCode) - ;[server, stopServer] = await getServer() - eventsProcessor = new EventsProcessor(server) - team = (await server.db.postgresQuery('SELECT * FROM posthog_team LIMIT 1')).rows[0] - now = DateTime.utc() -}) - -afterEach(async () => { - await stopServer?.() -}) - async function getEvents(): Promise { const result = await server.db.postgresQuery('SELECT * FROM posthog_event') return result.rows as Event[] @@ -84,40 +53,60 @@ async function createPerson(team: Team, distinctIds: string[]) { return person } -test('capture no element', async () => { - await createPerson(team, ['asdfasdfasdf']) - - await eventsProcessor.processEvent( - 'asdfasdfasdf', - '', - '', - ({ - event: '$pageview', - properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - DateTime.utc(), - DateTime.utc(), - new UUIDT().toString() - ) +describe('process event', () => { + beforeEach(async () => { + const testCode = ` + function processEvent (event, meta) { + event.properties["somewhere"] = "over the rainbow"; + return event + } + ` + await resetTestDatabase(testCode) + ;[server, stopServer] = await getServer() + eventsProcessor = new EventsProcessor(server) + team = (await server.db.postgresQuery('SELECT * FROM posthog_team LIMIT 1')).rows[0] + now = DateTime.utc() + }) - expect(await getDistinctIds((await getPersons())[0])).toEqual(['asdfasdfasdf']) - const [event] = await getEvents() - expect(event.event).toBe('$pageview') -}) + afterEach(async () => { + await stopServer?.() + }) -test('long event name substr', async () => { - await eventsProcessor.processEvent( - 'xxx', - '', - '', - ({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, - team.id, - DateTime.utc(), - DateTime.utc(), - 'uuid' - ) + test('capture no element', async () => { + await createPerson(team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + + expect(await getDistinctIds((await getPersons())[0])).toEqual(['asdfasdfasdf']) + const [event] = await getEvents() + expect(event.event).toBe('$pageview') + }) - const [event] = await getEvents() - expect(event.event?.length).toBe(200) + test('long event name substr', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + 'uuid' + ) + + const [event] = await getEvents() + expect(event.event?.length).toBe(200) + }) }) From f3572ce1fb4ee619b4bb0d35b7333ad31db5b759 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 11:58:31 +0100 Subject: [PATCH 07/47] add more tests --- src/types.ts | 4 +- tests/postgres/process-event.test.ts | 241 +++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 2 deletions(-) diff --git a/src/types.ts b/src/types.ts index 159b0ac3..7809796b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -238,11 +238,11 @@ export interface Event { event?: string properties: Record elements?: Element[] - timestamp: string + timestamp: Date team_id: number distinct_id: string elements_hash: string - created_at: DateTime + created_at: Date } /** Properties shared by RawPerson and Person. */ diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index bc57b4dd..c3c9e7b9 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -72,6 +72,10 @@ describe('process event', () => { await stopServer?.() }) + test.skip('capture new person', async () => { + // TODO + }) + test('capture no element', async () => { await createPerson(team, ['asdfasdfasdf']) @@ -94,6 +98,243 @@ describe('process event', () => { expect(event.event).toBe('$pageview') }) + test('capture sent_at', async () => { + await createPerson(team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await getEvents() + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromJSDate(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture sent_at no timezones', async () => { + await createPerson(team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }).setZone('UTC+4') + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }).setZone('UTC+4') + + // TODO: not sure if this is correct? + // tomorrow = tomorrow.replace(tzinfo=None) + // tomorrow_sent_at = tomorrow_sent_at.replace(tzinfo=None) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow, + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await getEvents() + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromJSDate(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture no sent_at', async () => { + await createPerson(team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + null, + new UUIDT().toString() + ) + + const [event] = await getEvents() + const difference = tomorrow.diff(DateTime.fromJSDate(event.timestamp), 'seconds').seconds + expect(difference).toBeLessThan(1) + }) + + test('ip capture', async () => { + await createPerson(team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + const [event] = await getEvents() + expect(event.properties['$ip']).toBe('11.12.13.14') + }) + + test('ip override', async () => { + await createPerson(team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { $ip: '1.0.0.1', distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + const [event] = await getEvents() + expect(event.properties['$ip']).toBe('1.0.0.1') + }) + + test('anonymized ip capture', async () => { + await server.db.postgresQuery('update posthog_team set anonymize_ips = $1', [true]) + await createPerson(team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + const [event] = await getEvents() + expect(event.properties['$ip']).not.toBeDefined() + }) + + test.skip('alias', async () => { + // TODO + }) + + test.skip('alias reverse', async () => { + // TODO + }) + + test.skip('alias twice', async () => { + // TODO + }) + + test.skip('alias before person', async () => { + // TODO + }) + + test.skip('alias both existing', async () => { + // TODO + }) + + test.skip('offset timestamp', async () => { + // TODO + }) + + test.skip('offset timestamp no sent_at', async () => { + // TODO + }) + + test.skip('alias merge properties', async () => { + // TODO + }) + + test.skip('long htext', async () => { + // TODO + }) + + test.skip('capture first team event', async () => { + // TODO + }) + + test.skip('snapshot event stored as session_recording_event', async () => { + // TODO + }) + + test.skip('identify set', async () => { + // TODO + }) + + test.skip('identify set_once', async () => { + // TODO + }) + + test.skip('distinct with anonymous_id', async () => { + // TODO + }) + + test.skip('distinct with anonymous_id which was already created', async () => { + // TODO + }) + + test.skip('distinct with multiple anonymous_ids which were already created', async () => { + // TODO + }) + + test.skip('distinct team leakage', async () => { + // TODO + }) + + test.skip('set is_identified', async () => { + // TODO + }) + + test.skip('team event_properties', async () => { + // TODO + }) + + test.skip('add feature flags if missing', async () => { + // TODO + }) + + test.skip('event name dict json', async () => { + // TODO + }) + + test.skip('event name list json', async () => { + // TODO + }) + test('long event name substr', async () => { await eventsProcessor.processEvent( 'xxx', From 460b331740b64b235487580e10c2d39d3793b804 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 12:02:46 +0100 Subject: [PATCH 08/47] opt out of posthog in test mode --- src/ingestion/process-event.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 2f3fe4df..0a58575b 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -28,6 +28,9 @@ export class EventsProcessor { this.kafkaProducer = pluginsServer.kafkaProducer! this.celery = new Client(pluginsServer.redis) this.posthog = nodePostHog('sTMFPsFhdP1Ssg') + if (process.env.NODE_ENV === 'test') { + this.posthog.optOut() + } } public async processEvent( From 6c8edf043e87ec8d9b01ec62a1c62555a42a9b71 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 12:02:53 +0100 Subject: [PATCH 09/47] add first alias test --- tests/postgres/process-event.test.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index c3c9e7b9..fb30c6c2 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -247,8 +247,25 @@ describe('process event', () => { expect(event.properties['$ip']).not.toBeDefined() }) - test.skip('alias', async () => { - // TODO + test('alias', async () => { + await createPerson(team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + expect(getDistinctIds((await getPersons())[0])).toBe(['old_distinct_id', 'new_distinct_id']) }) test.skip('alias reverse', async () => { From 4c27e2c617ffccfa96d4e4cf321aba70c66ab532 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:20:07 +0100 Subject: [PATCH 10/47] always use UTC times when talking to postgres --- src/server.ts | 13 +++++++++++++ src/types.ts | 4 ++-- src/utils.ts | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/server.ts b/src/server.ts index 781b2341..71d4432d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -19,6 +19,19 @@ import { status } from './status' import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' +import { types } from 'pg' +import { DateTime } from 'luxon' + +// postgres will only return strings as dates +types.setTypeParser(1083 /* types.TypeId.TIME */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null +) +types.setTypeParser(1114 /* types.TypeId.TIMESTAMP */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null +) +types.setTypeParser(1184 /* types.TypeId.TIMESTAMPTZ */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null +) export async function createServer( config: Partial = {}, diff --git a/src/types.ts b/src/types.ts index 87707454..903cc089 100644 --- a/src/types.ts +++ b/src/types.ts @@ -248,11 +248,11 @@ export interface Event { event?: string properties: Record elements?: Element[] - timestamp: Date + timestamp: string team_id: number distinct_id: string elements_hash: string - created_at: Date + created_at: string } /** Properties shared by RawPerson and Person. */ diff --git a/src/utils.ts b/src/utils.ts index e91ac52a..00e851ca 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -286,7 +286,7 @@ export function castTimestampOrNow(timestamp?: DateTime | string | null): string } else if (typeof timestamp === 'string') { timestamp = DateTime.fromISO(timestamp) } - return timestamp.toUTC().toFormat('yyyy-MM-dd HH:mm:ss.u') + return timestamp.toUTC().toISO() } export function delay(ms: number): Promise { From 2f6c1a7af4e695368f6723cfe551bc0d55919d6b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:20:22 +0100 Subject: [PATCH 11/47] prevent a crash --- src/db.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/db.ts b/src/db.ts index 5a64697b..b47f7b42 100644 --- a/src/db.ts +++ b/src/db.ts @@ -45,8 +45,10 @@ export class DB { AND posthog_persondistinctid.distinct_id = $2`, [teamId, distinctId] ) - const rawPerson: RawPerson = selectResult.rows[0] - return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } + if (selectResult.rows.length > 0) { + const rawPerson: RawPerson = selectResult.rows[0] + return { ...rawPerson, created_at: DateTime.fromISO(rawPerson.created_at) } + } } public async createPerson( From a1b25d08ebc0407578f998648ceb0fda2aa63fc1 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:20:33 +0100 Subject: [PATCH 12/47] bit of clarity to help debug --- src/db.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db.ts b/src/db.ts index b47f7b42..e3a89567 100644 --- a/src/db.ts +++ b/src/db.ts @@ -82,11 +82,12 @@ export class DB { public async updatePerson(person: Person, update: Partial): Promise { const updatedPerson: Person = { ...person, ...update } + const values = [...Object.values(unparsePersonPartial(update)), person.id] await this.postgresQuery( `UPDATE posthog_person SET ${Object.keys(update).map( (field, index) => sanitizeSqlIdentifier(field) + ' = $' + (index + 1) )} WHERE id = $${Object.values(update).length + 1}`, - [...Object.values(unparsePersonPartial(update)), person.id] + values ) if (this.kafkaProducer) { const data = { From 90d859f9a8e6148ec5f65f7dffaaa4a75eaf9b46 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:20:45 +0100 Subject: [PATCH 13/47] fix bug with table name --- src/db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.ts b/src/db.ts index e3a89567..2ca8821d 100644 --- a/src/db.ts +++ b/src/db.ts @@ -106,7 +106,7 @@ export class DB { } public async deletePerson(personId: number): Promise { - await this.postgresQuery('DELETE FROM person_distinct_id WHERE person_id = $1', [personId]) + await this.postgresQuery('DELETE FROM posthog_persondistinctid WHERE person_id = $1', [personId]) await this.postgresQuery('DELETE FROM posthog_person WHERE id = $1', [personId]) if (this.clickhouse) { await this.clickhouse.query(`ALTER TABLE person DELETE WHERE id = ${personId}`).toPromise() From 0b986836fcdb4f07aeb5eb2beb18e13fba6697e3 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:20:56 +0100 Subject: [PATCH 14/47] fix bug with passing object instead of id --- src/ingestion/process-event.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 0a58575b..2ff5193b 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -266,7 +266,7 @@ export class EventsProcessor { const otherPersonDistinctIds: PersonDistinctId[] = ( await this.db.postgresQuery( 'SELECT * FROM posthog_persondistinctid WHERE person_id = $1 AND team_id = $2', - [otherPerson, mergeInto.team_id] + [otherPerson.id, mergeInto.team_id] ) ).rows for (const personDistinctId of otherPersonDistinctIds) { From 5fab1e25d6375610c9bafa17f09d47a59383e253 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:21:21 +0100 Subject: [PATCH 15/47] add some alias tests (all green now) --- tests/postgres/process-event.test.ts | 94 ++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index fb30c6c2..14802bbc 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -38,7 +38,7 @@ async function getPersons(): Promise { async function getDistinctIds(person: Person) { const result = await server.db.postgresQuery( - 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2', + 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', [person.id, person.team_id] ) return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) @@ -121,7 +121,7 @@ describe('process event', () => { ) const [event] = await getEvents() - const eventSecondsBeforeNow = rightNow.diff(DateTime.fromJSDate(event.timestamp), 'seconds').seconds + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds expect(eventSecondsBeforeNow).toBeGreaterThan(590) expect(eventSecondsBeforeNow).toBeLessThan(610) @@ -154,7 +154,7 @@ describe('process event', () => { ) const [event] = await getEvents() - const eventSecondsBeforeNow = rightNow.diff(DateTime.fromJSDate(event.timestamp), 'seconds').seconds + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds expect(eventSecondsBeforeNow).toBeGreaterThan(590) expect(eventSecondsBeforeNow).toBeLessThan(610) @@ -182,7 +182,7 @@ describe('process event', () => { ) const [event] = await getEvents() - const difference = tomorrow.diff(DateTime.fromJSDate(event.timestamp), 'seconds').seconds + const difference = tomorrow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds expect(difference).toBeLessThan(1) }) @@ -265,19 +265,91 @@ describe('process event', () => { ) expect((await getEvents()).length).toBe(1) - expect(getDistinctIds((await getPersons())[0])).toBe(['old_distinct_id', 'new_distinct_id']) + expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) }) - test.skip('alias reverse', async () => { - // TODO + test('alias reverse', async () => { + await createPerson(team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'old_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'old_distinct_id', token: team.api_token, alias: 'new_distinct_id' }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) }) - test.skip('alias twice', async () => { - // TODO + test('alias twice', async () => { + await createPerson(team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) + + await createPerson(team, ['old_distinct_id_2']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id_2' }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + expect((await getEvents()).length).toBe(2) + expect(await getDistinctIds((await getPersons())[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + 'old_distinct_id_2', + ]) }) - test.skip('alias before person', async () => { - // TODO + test('alias before person', async () => { + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + expect((await getPersons()).length).toBe(1) + expect(await getDistinctIds((await getPersons())[0])).toEqual(['new_distinct_id', 'old_distinct_id']) }) test.skip('alias both existing', async () => { From 60bf596b6828e4494323acba12ccffe62921377b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:34:25 +0100 Subject: [PATCH 16/47] save merged properties --- src/ingestion/process-event.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 2ff5193b..2b758770 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -259,7 +259,7 @@ export class EventsProcessor { } } - await this.db.updatePerson(mergeInto, { created_at: firstSeen }) + await this.db.updatePerson(mergeInto, { created_at: firstSeen, properties: mergeInto.properties }) // merge the distinct_ids for (const otherPerson of peopleToMerge) { From 92349255ddb4f4e3bfea93bd716bcb7d3979b99b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:35:08 +0100 Subject: [PATCH 17/47] few more tests --- tests/postgres/process-event.test.ts | 135 +++++++++++++++++++++------ 1 file changed, 107 insertions(+), 28 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 14802bbc..8ae6e16e 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -44,8 +44,15 @@ async function getDistinctIds(person: Person) { return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) } -async function createPerson(team: Team, distinctIds: string[]) { - const person = await server.db.createPerson(DateTime.utc(), {}, team.id, null, false, new UUIDT().toString()) +async function createPerson(team: Team, distinctIds: string[], properties: Record = {}) { + const person = await server.db.createPerson( + DateTime.utc(), + properties, + team.id, + null, + false, + new UUIDT().toString() + ) for (const distinctId of distinctIds) { await server.db.addDistinctId(person, distinctId) } @@ -88,8 +95,8 @@ describe('process event', () => { properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) @@ -198,8 +205,8 @@ describe('process event', () => { properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) const [event] = await getEvents() @@ -218,8 +225,8 @@ describe('process event', () => { properties: { $ip: '1.0.0.1', distinct_id: 'asdfasdfasdf', token: team.api_token }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) const [event] = await getEvents() @@ -239,8 +246,8 @@ describe('process event', () => { properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) const [event] = await getEvents() @@ -259,8 +266,8 @@ describe('process event', () => { properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) @@ -280,8 +287,8 @@ describe('process event', () => { properties: { distinct_id: 'old_distinct_id', token: team.api_token, alias: 'new_distinct_id' }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) @@ -301,8 +308,8 @@ describe('process event', () => { properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) @@ -320,8 +327,8 @@ describe('process event', () => { properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id_2' }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) expect((await getEvents()).length).toBe(2) @@ -342,8 +349,8 @@ describe('process event', () => { properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, } as any) as PluginEvent, team.id, - DateTime.utc(), - DateTime.utc(), + now, + now, new UUIDT().toString() ) @@ -352,20 +359,92 @@ describe('process event', () => { expect(await getDistinctIds((await getPersons())[0])).toEqual(['new_distinct_id', 'old_distinct_id']) }) - test.skip('alias both existing', async () => { - // TODO + test('alias both existing', async () => { + await createPerson(team, ['old_distinct_id']) + await createPerson(team, ['new_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) }) - test.skip('offset timestamp', async () => { - // TODO + test('offset timestamp', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents()).length).toBe(1) + + const [event] = await getEvents() + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') }) - test.skip('offset timestamp no sent_at', async () => { - // TODO + test('offset timestamp no sent_at', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + null, + new UUIDT().toString() + ) + expect((await getEvents()).length).toBe(1) + + const [event] = await getEvents() + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') }) - test.skip('alias merge properties', async () => { - // TODO + test('alias merge properties', async () => { + await createPerson(team, ['old_distinct_id'], { key_on_both: 'old value both', key_on_old: 'old value' }) + await createPerson(team, ['new_distinct_id'], { key_on_both: 'new value both', key_on_new: 'new value' }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + const [person] = await getPersons() + expect(await getDistinctIds(person)).toEqual(['old_distinct_id', 'new_distinct_id']) + expect(person.properties).toEqual({ + key_on_both: 'new value both', + key_on_new: 'new value', + key_on_old: 'old value', + }) }) test.skip('long htext', async () => { From e63a254b5afb733a84de1c07afbef7f15cf2fc9d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 13:57:28 +0100 Subject: [PATCH 18/47] more missing tests --- tests/postgres/process-event.test.ts | 347 ++++++++++++++++++++++++--- 1 file changed, 317 insertions(+), 30 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 8ae6e16e..39dd8ffb 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -79,8 +79,8 @@ describe('process event', () => { await stopServer?.() }) - test.skip('capture new person', async () => { - // TODO + test('capture new person', async () => { + expect(true).toBe(false) }) test('capture no element', async () => { @@ -438,6 +438,7 @@ describe('process event', () => { ) expect((await getEvents()).length).toBe(1) + expect((await getPersons()).length).toBe(1) const [person] = await getPersons() expect(await getDistinctIds(person)).toEqual(['old_distinct_id', 'new_distinct_id']) expect(person.properties).toEqual({ @@ -448,59 +449,345 @@ describe('process event', () => { }) test.skip('long htext', async () => { - // TODO - }) + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 'new_distinct_id', + token: team.api_token, + $elements: [ + { + tag_name: 'a', + $el_text: 'a'.repeat(2050), + attr__href: 'a'.repeat(2050), + nth_child: 1, + nth_of_type: 2, + attr__class: 'btn btn-sm', + }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) - test.skip('capture first team event', async () => { - // TODO + // event = get_events()[0] + // element = get_elements(event.id)[0] + // self.assertEqual(len(element.href), 2048) + // self.assertEqual(len(element.text), 400) + expect(true).toBe(false) }) - test.skip('snapshot event stored as session_recording_event', async () => { - // TODO + test('capture first team event', async () => { + expect(true).toBe(false) }) - test.skip('identify set', async () => { - // TODO + test('snapshot event stored as session_recording_event', async () => { + expect(true).toBe(false) }) - test.skip('identify set_once', async () => { - // TODO + test('identify set', async () => { + await createPerson(team, ['distinct_id']) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + + const [event] = await getEvents() + expect(event.properties['$set']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await getPersons() + expect(await getDistinctIds(person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents()).length).toBe(2) + const [person2] = await getPersons() + expect(person2.properties).toEqual({ a_prop: 'test-2', b_prop: 'test-2b', c_prop: 'test-1' }) }) - test.skip('distinct with anonymous_id', async () => { - // TODO + test('identify set_once', async () => { + await createPerson(team, ['distinct_id']) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + + const [event] = await getEvents() + expect(event.properties['$set_once']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await getPersons() + expect(await getDistinctIds(person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents()).length).toBe(2) + const [person2] = await getPersons() + expect(person2.properties).toEqual({ a_prop: 'test-1', b_prop: 'test-2b', c_prop: 'test-1' }) }) - test.skip('distinct with anonymous_id which was already created', async () => { - // TODO + test('distinct with anonymous_id', async () => { + await createPerson(team, ['anonymous_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents()).length).toBe(1) + const [event] = await getEvents() + expect(event.properties['$set']).toEqual({ a_prop: 'test' }) + const [person] = await getPersons() + expect(await getDistinctIds(person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test' }) + + // check no errors as this call can happen multiple times + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) }) - test.skip('distinct with multiple anonymous_ids which were already created', async () => { - // TODO + // This case is likely to happen after signup, for example: + // 1. User browses website with anonymous_id + // 2. User signs up, triggers event with their new_distinct_id (creating a new Person) + // 3. In the frontend, try to alias anonymous_id with new_distinct_id + // Result should be that we end up with one Person with both ID's + test('distinct with anonymous_id which was already created', async () => { + await createPerson(team, ['anonymous_id']) + await createPerson(team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person] = await getPersons() + expect(await getDistinctIds(person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties['email']).toEqual('someone@gmail.com') }) - test.skip('distinct team leakage', async () => { - // TODO + test('distinct with multiple anonymous_ids which were already created', async () => { + await createPerson(team, ['anonymous_id']) + await createPerson(team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person] = await getPersons() + expect(await getDistinctIds(person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties['email']).toEqual('someone@gmail.com') + + await createPerson(team, ['anonymous_id_2']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id_2', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const persons = await getPersons() + expect(persons.length).toBe(1) + const person2 = persons[0] + expect(await getDistinctIds(person2)).toEqual(['anonymous_id', 'new_distinct_id', 'anonymous_id_2']) + expect(person2.properties['email']).toEqual('someone@gmail.com') }) - test.skip('set is_identified', async () => { - // TODO + test('distinct team leakage', async () => { + expect(true).toBe(false) }) - test.skip('team event_properties', async () => { - // TODO + test('set is_identified', async () => { + const distinct_id = '777' + const person1 = await createPerson(team, [distinct_id]) + expect(person1.is_identified).toBe(false) + + await eventsProcessor.processEvent( + distinct_id, + '', + '', + ({ event: '$identify', properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person2] = await getPersons() + expect(person2.is_identified).toBe(true) }) - test.skip('add feature flags if missing', async () => { - // TODO + test('team event_properties', async () => { + expect(true).toBe(false) }) - test.skip('event name dict json', async () => { - // TODO + test('event name object json', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: { 'event name': 'as object' }, properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents() + expect(event.event).toEqual('{"event name":"as object"}') }) - test.skip('event name list json', async () => { - // TODO + test('event name array json', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: ['event name', 'a list'], properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents() + expect(event.event).toEqual('["event name","a list"]') }) test('long event name substr', async () => { From 0ad5c14cd490835eb01a3d0d2757afbea89c2e11 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 14:03:15 +0100 Subject: [PATCH 19/47] fix test --- tests/postgres/process-event.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 39dd8ffb..7c80e84e 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -701,9 +701,10 @@ describe('process event', () => { new UUIDT().toString() ) - const [person] = await getPersons() - expect(await getDistinctIds(person)).toEqual(['anonymous_id', 'new_distinct_id']) - expect(person.properties['email']).toEqual('someone@gmail.com') + const persons1 = await getPersons() + expect(persons1.length).toBe(1) + expect(await getDistinctIds(persons1[0])).toEqual(['anonymous_id', 'new_distinct_id']) + expect(persons1[0].properties['email']).toEqual('someone@gmail.com') await createPerson(team, ['anonymous_id_2']) @@ -725,11 +726,10 @@ describe('process event', () => { new UUIDT().toString() ) - const persons = await getPersons() - expect(persons.length).toBe(1) - const person2 = persons[0] - expect(await getDistinctIds(person2)).toEqual(['anonymous_id', 'new_distinct_id', 'anonymous_id_2']) - expect(person2.properties['email']).toEqual('someone@gmail.com') + const persons2 = await getPersons() + expect(persons2.length).toBe(1) + expect(await getDistinctIds(persons2[0])).toEqual(['anonymous_id', 'new_distinct_id', 'anonymous_id_2']) + expect(persons2[0].properties['email']).toEqual('someone@gmail.com') }) test('distinct team leakage', async () => { From f02556616d21f2301eabfca667c9e3786938076b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 14:06:35 +0100 Subject: [PATCH 20/47] team event properties test --- tests/postgres/process-event.test.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 7c80e84e..fca66d66 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -44,6 +44,10 @@ async function getDistinctIds(person: Person) { return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) } +async function getFirstTeam(): Promise { + return (await server.db.postgresQuery('SELECT * FROM posthog_team LIMIT 1')).rows[0] +} + async function createPerson(team: Team, distinctIds: string[], properties: Record = {}) { const person = await server.db.createPerson( DateTime.utc(), @@ -71,7 +75,7 @@ describe('process event', () => { await resetTestDatabase(testCode) ;[server, stopServer] = await getServer() eventsProcessor = new EventsProcessor(server) - team = (await server.db.postgresQuery('SELECT * FROM posthog_team LIMIT 1')).rows[0] + team = await getFirstTeam() now = DateTime.utc() }) @@ -757,7 +761,22 @@ describe('process event', () => { }) test('team event_properties', async () => { - expect(true).toBe(false) + expect(team.event_properties_numerical).toEqual([]) + + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: 'purchase', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + team = await getFirstTeam() + expect(team.event_properties).toEqual(['price', 'name', '$ip']) + expect(team.event_properties_numerical).toEqual(['price']) }) test('event name object json', async () => { From e4a069b82c48d9a1bb7e4abdcc3dafb8439f410f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 14:17:42 +0100 Subject: [PATCH 21/47] fix bug --- src/ingestion/process-event.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 2b758770..d6626da4 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -398,7 +398,7 @@ export class EventsProcessor { JSON.stringify(team.event_names), JSON.stringify(team.event_names_with_usage), JSON.stringify(team.event_properties), - JSON.stringify(team.event_names_with_usage), + JSON.stringify(team.event_properties_with_usage), JSON.stringify(team.event_properties_numerical), team.id, ] From e478825141020bcdf51d09dafb26ab975057cdab Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 14:18:35 +0100 Subject: [PATCH 22/47] another test (partial) --- tests/helpers/sql.ts | 2 +- tests/postgres/process-event.test.ts | 65 +++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 99e3b2c9..ff5a41e4 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -29,7 +29,7 @@ export async function resetTestDatabase(code: string): Promise { organization_id: commonOrganizationId, app_urls: [], name: 'TEST PROJECT', - event_names: JSON.stringify(['test']), + event_names: JSON.stringify([]), event_names_with_usage: JSON.stringify([]), event_properties: JSON.stringify([]), event_properties_with_usage: JSON.stringify([]), diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index fca66d66..1ad011ad 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,6 +1,6 @@ import { PluginEvent, Properties } from '@posthog/plugin-scaffold/src/types' import { createServer } from '../../src/server' -import { LogLevel, PluginsServer, Team, Event, Person, PersonDistinctId } from '../../src/types' +import { LogLevel, PluginsServer, Team, Event, Person, PersonDistinctId, Element } from '../../src/types' import { resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' @@ -48,6 +48,10 @@ async function getFirstTeam(): Promise { return (await server.db.postgresQuery('SELECT * FROM posthog_team LIMIT 1')).rows[0] } +async function getElements(event: Event): Promise { + return [] +} + async function createPerson(team: Team, distinctIds: string[], properties: Record = {}) { const person = await server.db.createPerson( DateTime.utc(), @@ -84,7 +88,64 @@ describe('process event', () => { }) test('capture new person', async () => { - expect(true).toBe(false) + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [true, team.id]) + team = await getFirstTeam() + + expect(team.event_names).toEqual([]) + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + // TODO: add this back? + // num_queries = 28 + // if settings.EE_AVAILABLE: # extra queries to check for hooks + // num_queries += 4 + // if settings.MULTI_TENANCY: # extra query to check for billing plan + // num_queries += 1 + // with self.assertNumQueries(num_queries): + + const [event] = await getEvents() + const [person] = await getPersons() + const distinctIds = await getDistinctIds(person) + + expect(event.distinct_id).toEqual('2') + expect(distinctIds).toEqual(['2']) + expect(event.event).toEqual('$autocapture') + + // TODO: add this + // const elements = await getElements(event) + // expect(elements[0].tag_name).toEqual('a') + // expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) + // expect(elements[1].order).toEqual(1) + // expect(elements[1].text).toEqual('💻') + + team = await getFirstTeam() + expect(team.event_names).toEqual(['$autocapture']) + expect(team.event_names_with_usage).toEqual([{ event: '$autocapture', volume: null, usage_count: null }]) + expect(team.event_properties).toEqual(['distinct_id', 'token', '$ip']) + expect(team.event_properties_with_usage).toEqual([ + { key: 'distinct_id', usage_count: null, volume: null }, + { key: 'token', usage_count: null, volume: null }, + { key: '$ip', usage_count: null, volume: null }, + ]) }) test('capture no element', async () => { From c2b9c7abdd9643b745e785aff27062685de1bf64 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 14:39:01 +0100 Subject: [PATCH 23/47] clarify postgres magic --- src/server.ts | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/server.ts b/src/server.ts index 71d4432d..8deedd40 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,4 +1,4 @@ -import { Pool } from 'pg' +import { Pool, types as pgTypes } from 'pg' import * as schedule from 'node-schedule' import Redis from 'ioredis' import { Kafka, logLevel, Producer } from 'kafkajs' @@ -19,20 +19,8 @@ import { status } from './status' import { startSchedule } from './services/schedule' import { ConnectionOptions } from 'tls' import { DB } from './db' -import { types } from 'pg' import { DateTime } from 'luxon' -// postgres will only return strings as dates -types.setTypeParser(1083 /* types.TypeId.TIME */, (timeStr) => - timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null -) -types.setTypeParser(1114 /* types.TypeId.TIMESTAMP */, (timeStr) => - timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null -) -types.setTypeParser(1184 /* types.TypeId.TIMESTAMPTZ */, (timeStr) => - timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null -) - export async function createServer( config: Partial = {}, threadId: number | null = null @@ -101,6 +89,19 @@ export async function createServer( kafkaProducer = kafka.producer() } + // `node-postgres` will return dates as plain JS Date objects, which will use the local timezone. + // This converts all date fields to a proper luxon UTC DateTime + // Unfortunately this must be done on a global object before initializing the `Pool` + pgTypes.setTypeParser(1083 /* types.TypeId.TIME */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + pgTypes.setTypeParser(1114 /* types.TypeId.TIMESTAMP */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + pgTypes.setTypeParser(1184 /* types.TypeId.TIMESTAMPTZ */, (timeStr) => + timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null + ) + const postgres = new Pool({ connectionString: serverConfig.DATABASE_URL, ssl: process.env.DEPLOYMENT?.startsWith('Heroku') From 137dcde9e674b1f760543715e360177d6093f187 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 14:48:24 +0100 Subject: [PATCH 24/47] different timestamp format for creating event in clickhouse & postgresql --- src/ingestion/process-event.ts | 18 +++++++++++++++--- src/types.ts | 5 +++++ src/utils.ts | 11 +++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index d6626da4..892ca83e 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -1,11 +1,20 @@ import { PluginEvent, Properties } from '@posthog/plugin-scaffold' import { DateTime, Duration } from 'luxon' -import { PluginsServer, Element, Team, Person, PersonDistinctId, CohortPeople, SessionRecordingEvent } from '../types' +import { + CohortPeople, + Element, + Person, + PersonDistinctId, + PluginsServer, + SessionRecordingEvent, + Team, + TimestampFormat, +} from '../types' import { castTimestampOrNow, UUIDT } from '../utils' import { Event as EventProto, IEvent } from '../idl/protos' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' -import { sanitizeEventName, elementsToString } from './utils' +import { elementsToString, sanitizeEventName } from './utils' import { ClickHouse } from 'clickhouse' import { DB } from '../db' import { status } from '../status' @@ -416,7 +425,10 @@ export class EventsProcessor { elements?: Element[], siteUrl?: string ): Promise { - const timestampString = castTimestampOrNow(timestamp) + const timestampString = castTimestampOrNow( + timestamp, + this.kafkaProducer ? TimestampFormat.Clickhouse : TimestampFormat.ISO + ) const elementsChain = elements && elements.length ? elementsToString(elements) : '' const data: IEvent = { diff --git a/src/types.ts b/src/types.ts index 903cc089..cfda6a97 100644 --- a/src/types.ts +++ b/src/types.ts @@ -298,3 +298,8 @@ export interface SessionRecordingEvent { snapshot_data: string created_at: string } + +export enum TimestampFormat { + Clickhouse = 'clickhouse', + ISO = 'iso', +} diff --git a/src/utils.ts b/src/utils.ts index 00e851ca..c166d03b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,7 +2,7 @@ import { Readable } from 'stream' import * as tar from 'tar-stream' import AdmZip from 'adm-zip' import * as zlib from 'zlib' -import { LogLevel } from './types' +import { LogLevel, TimestampFormat } from './types' import { randomBytes } from 'crypto' import { DateTime } from 'luxon' import { status } from './status' @@ -280,12 +280,19 @@ export class UUIDT extends UUID { } /** Format timestamp for ClickHouse. */ -export function castTimestampOrNow(timestamp?: DateTime | string | null): string { +export function castTimestampOrNow( + timestamp?: DateTime | string | null, + timestampFormat: TimestampFormat = TimestampFormat.ISO +): string { if (!timestamp) { timestamp = DateTime.utc() } else if (typeof timestamp === 'string') { timestamp = DateTime.fromISO(timestamp) } + + if (timestampFormat === TimestampFormat.Clickhouse) { + return timestamp.toUTC().toFormat('yyyy-MM-dd HH:mm:ss.u') + } return timestamp.toUTC().toISO() } From e78889237781bcbe4a72346e2a183a180d739aa7 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 14:53:28 +0100 Subject: [PATCH 25/47] make element tests fail --- tests/postgres/process-event.test.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 1ad011ad..b932bd73 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -130,12 +130,11 @@ describe('process event', () => { expect(distinctIds).toEqual(['2']) expect(event.event).toEqual('$autocapture') - // TODO: add this - // const elements = await getElements(event) - // expect(elements[0].tag_name).toEqual('a') - // expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) - // expect(elements[1].order).toEqual(1) - // expect(elements[1].text).toEqual('💻') + const elements = await getElements(event) + expect(elements[0].tag_name).toEqual('a') + expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) + expect(elements[1].order).toEqual(1) + expect(elements[1].text).toEqual('💻') team = await getFirstTeam() expect(team.event_names).toEqual(['$autocapture']) @@ -541,11 +540,10 @@ describe('process event', () => { new UUIDT().toString() ) - // event = get_events()[0] - // element = get_elements(event.id)[0] - // self.assertEqual(len(element.href), 2048) - // self.assertEqual(len(element.text), 400) - expect(true).toBe(false) + const [event] = await getEvents() + const [element] = await getElements(event) + expect(element.href?.length).toEqual(2048) + expect(element.text?.length).toEqual(400) }) test('capture first team event', async () => { From cc743cb0f195b6444fb4a624e7fa282d7d91d734 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 22:44:59 +0100 Subject: [PATCH 26/47] capture first team event test --- tests/helpers/plugins.ts | 2 ++ tests/helpers/sql.ts | 23 ++++++++++++++++++- tests/postgres/process-event.test.ts | 33 +++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tests/helpers/plugins.ts b/tests/helpers/plugins.ts index 75f89e9b..224f3840 100644 --- a/tests/helpers/plugins.ts +++ b/tests/helpers/plugins.ts @@ -4,6 +4,8 @@ import path from 'path' import os from 'os' import AdmZip from 'adm-zip' +export const commonUserId = 1001 +export const commonOrganizationMembershipId = '0177364a-fc7b-0000-511c-137090b9e4e1' export const commonOrganizationId = 'ca30f2ec-e9a4-4001-bf27-3ef194086068' export const plugin60: Plugin = { diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index ff5a41e4..b869b366 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -1,4 +1,4 @@ -import { makePluginObjects, commonOrganizationId } from './plugins' +import { makePluginObjects, commonOrganizationId, commonUserId, commonOrganizationMembershipId } from './plugins' import { defaultConfig } from '../../src/config' import { Pool } from 'pg' import { delay, UUIDT } from '../../src/utils' @@ -14,15 +14,36 @@ export async function resetTestDatabase(code: string): Promise { await db.query('DELETE FROM posthog_pluginconfig') await db.query('DELETE FROM posthog_plugin') await db.query('DELETE FROM posthog_team') + await db.query('DELETE FROM posthog_organizationmembership') await db.query('DELETE FROM posthog_organization') + await db.query('DELETE FROM posthog_user') const teamIds = mocks.pluginConfigRows.map((c) => c.team_id) + await insertRow(db, 'posthog_user', { + id: commonUserId, + password: 'gibberish', + first_name: 'PluginTest', + last_name: 'User', + email: 'test@posthog.com', + distinct_id: 'plugin_test_user_distinct_id', + is_staff: false, + is_active: false, + date_joined: new Date().toISOString(), + }) await insertRow(db, 'posthog_organization', { id: commonOrganizationId, name: 'TEST ORG', created_at: new Date().toISOString(), updated_at: new Date().toISOString(), }) + await insertRow(db, 'posthog_organizationmembership', { + id: commonOrganizationMembershipId, + organization_id: commonOrganizationId, + user_id: commonUserId, + level: 15, + joined_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }) for (const teamId of teamIds) { await insertRow(db, 'posthog_team', { id: teamId, diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 1ad011ad..8568dc16 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -549,7 +549,38 @@ describe('process event', () => { }) test('capture first team event', async () => { - expect(true).toBe(false) + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [false, team.id]) + + eventsProcessor.posthog = { + identify: jest.fn((distinctId) => true), + capture: jest.fn((event, properties) => true), + } as any + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 1, + token: team.api_token, + $elements: [{ tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(eventsProcessor.posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id') + expect(eventsProcessor.posthog.capture).toHaveBeenCalledWith('first team event ingested', { + team: team.uuid, + }) + + team = await getFirstTeam() + expect(team.ingested_event).toEqual(true) }) test('snapshot event stored as session_recording_event', async () => { From 11b274620907d773e6c6d89ae3961ae12cbda866 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 28 Jan 2021 23:15:16 +0100 Subject: [PATCH 27/47] insert session recording events --- src/ingestion/process-event.ts | 21 ++++++++++---- src/types.ts | 5 ++++ tests/helpers/sql.ts | 1 + tests/postgres/process-event.test.ts | 42 ++++++++++++++++++++++++++-- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 892ca83e..26f2b715 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -6,6 +6,7 @@ import { Person, PersonDistinctId, PluginsServer, + PostgresSessionRecordingEvent, SessionRecordingEvent, Team, TimestampFormat, @@ -494,7 +495,7 @@ export class EventsProcessor { session_id: string, timestamp: DateTime | string, snapshot_data: Record - ): Promise { + ): Promise { const timestampString = castTimestampOrNow(timestamp) const data: SessionRecordingEvent = { @@ -507,11 +508,19 @@ export class EventsProcessor { created_at: timestampString, } - await this.kafkaProducer.send({ - topic: KAFKA_SESSION_RECORDING_EVENTS, - messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }], - }) - + if (this.kafkaProducer) { + await this.kafkaProducer.send({ + topic: KAFKA_SESSION_RECORDING_EVENTS, + messages: [{ key: uuid, value: Buffer.from(JSON.stringify(data)) }], + }) + } else { + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_sessionrecordingevent (created_at, team_id, distinct_id, session_id, timestamp, snapshot_data) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', + [data.created_at, data.team_id, data.distinct_id, data.session_id, data.timestamp, data.snapshot_data] + ) + const eventCreated = insertResult.rows[0] as PostgresSessionRecordingEvent + return eventCreated + } return data } } diff --git a/src/types.ts b/src/types.ts index cfda6a97..48c1aa41 100644 --- a/src/types.ts +++ b/src/types.ts @@ -289,6 +289,7 @@ export interface CohortPeople { cohort_id: number person_id: number } + export interface SessionRecordingEvent { uuid: string timestamp: string @@ -299,6 +300,10 @@ export interface SessionRecordingEvent { created_at: string } +export interface PostgresSessionRecordingEvent extends Omit { + id: string +} + export enum TimestampFormat { Clickhouse = 'clickhouse', ISO = 'iso', diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index b869b366..9b7c2ca2 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -6,6 +6,7 @@ import { delay, UUIDT } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) const mocks = makePluginObjects(code) + await db.query('DELETE FROM posthog_sessionrecordingevent') await db.query('DELETE FROM posthog_persondistinctid') await db.query('DELETE FROM posthog_person') await db.query('DELETE FROM posthog_event') diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 8568dc16..ff54dd5d 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,6 +1,16 @@ import { PluginEvent, Properties } from '@posthog/plugin-scaffold/src/types' import { createServer } from '../../src/server' -import { LogLevel, PluginsServer, Team, Event, Person, PersonDistinctId, Element } from '../../src/types' +import { + LogLevel, + PluginsServer, + Team, + Event, + Person, + PersonDistinctId, + Element, + SessionRecordingEvent, + PostgresSessionRecordingEvent, +} from '../../src/types' import { resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' @@ -26,6 +36,11 @@ async function getServer(): Promise<[PluginsServer, () => Promise]> { return [server, stopServer] } +async function getSessionRecordingEvents(): Promise { + const result = await server.db.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') + return result.rows as PostgresSessionRecordingEvent[] +} + async function getEvents(): Promise { const result = await server.db.postgresQuery('SELECT * FROM posthog_event') return result.rows as Event[] @@ -584,7 +599,30 @@ describe('process event', () => { }) test('snapshot event stored as session_recording_event', async () => { - expect(true).toBe(false) + await eventsProcessor.processEvent( + 'some-id', + '', + '', + ({ + event: '$snapshot', + properties: { $session_id: 'abcf-efg', $snapshot_data: { timestamp: 123 } }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const events = await getEvents() + expect(events.length).toEqual(0) + + const sessionRecordingEvents = await getSessionRecordingEvents() + expect(sessionRecordingEvents.length).toBe(1) + + const [event] = sessionRecordingEvents + expect(event.session_id).toEqual('abcf-efg') + expect(event.distinct_id).toEqual('some-id') + expect(event.snapshot_data).toEqual({ timestamp: 123 }) }) test('identify set', async () => { From 23a2a6571bbbe3b4d700b23ff85f49be4e9b84d0 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 12:00:29 +0100 Subject: [PATCH 28/47] generate element hashes --- src/ingestion/process-event.ts | 18 +++++++--- src/ingestion/utils.ts | 49 ++++++++++++++++++++++++++++ tests/helpers/kafka.ts | 2 +- tests/postgres/process-event.test.ts | 7 +++- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 26f2b715..1f83477a 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -15,7 +15,7 @@ import { castTimestampOrNow, UUIDT } from '../utils' import { Event as EventProto, IEvent } from '../idl/protos' import { Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_SESSION_RECORDING_EVENTS } from './topics' -import { elementsToString, sanitizeEventName } from './utils' +import { elementsToString, hashElements, sanitizeEventName } from './utils' import { ClickHouse } from 'clickhouse' import { DB } from '../db' import { status } from '../status' @@ -454,9 +454,10 @@ export class EventsProcessor { ], }) } else { - // TODO: add element_group code! - // https://github.com/PostHog/posthog/blob/5d5ede19e4799dc71ffd5ec18e65bd969520b543/posthog/models/event.py#L235 - const elementsHash = '' + let elementsHash = '' + if (elements && elements.length > 0) { + elementsHash = await this.createElementGroup(elements) + } const insertResult = await this.db.postgresQuery( 'INSERT INTO posthog_event (created_at, event, distinct_id, properties, team_id, timestamp, elements, elements_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *', [ @@ -488,6 +489,15 @@ export class EventsProcessor { return data } + private async createElementGroup(elements: Element[]): Promise { + const cleanedElements = elements.map((element, index) => ({ ...element, order: index })) + const hash = hashElements(cleanedElements) + + // TODO: create actual element group and elements + + return hash + } + private async createSessionRecordingEvent( uuid: string, team_id: number, diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 8098cd1a..d8dd44e3 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -1,5 +1,6 @@ import { DateTime } from 'luxon' import { Element, BaseEventMessage, RawEventMessage, EventMessage, BasePerson, RawPerson, Person } from '../types' +import crypto from 'crypto' export function parseRawEventMessage(message: RawEventMessage): EventMessage { return { @@ -67,3 +68,51 @@ export function sanitizeEventName(eventName: any): string { } return eventName.substr(0, 200) } + +export function hashElements(elements: Element[]): string { + const elementsList = elements + .map((element) => ({ + attributes: element.attributes ?? null, + text: element.text ?? null, + tag_name: element.tag_name ?? null, + href: element.href ?? null, + attr_id: element.attr_id ?? null, + attr_class: element.attr_class ?? null, + nth_child: element.nth_child ?? null, + nth_of_type: element.nth_of_type ?? null, + order: element.order ?? null, + })) + .map((element) => { + const newElement: Record = {} + for (const key of Object.keys(element).sort()) { + newElement[key] = element[key as keyof typeof element] + } + return newElement + }) + + // escape utf-8 characters into `\u1234` + function jsonEscapeUTF(s: string): string { + return s.replace(/[^\x20-\x7F]/g, (x) => '\\u' + ('000' + x.codePointAt(0)?.toString(16)).slice(-4)) + } + + // produce output compatible to that of python's json.dumps + function pythonDumps(obj: any): string { + if (typeof obj === 'object' && obj !== null) { + if (Array.isArray(obj)) { + return `[${obj.map(pythonDumps).join(', ')}]` + } else { + return `{${Object.entries(obj) + .map(([k, v]) => `${JSON.stringify(k)}: ${pythonDumps(v)}`) + .join(', ')}}` + } + } else if (typeof obj === 'string') { + return jsonEscapeUTF(JSON.stringify(obj)) + } else { + return JSON.stringify(obj) + } + } + + const serializedString = pythonDumps(elementsList) + + return crypto.createHash('md5').update(serializedString).digest('hex') +} diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index 37b7dbe7..a7bb0368 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -16,7 +16,7 @@ export class KafkaObserver extends EventEmitter { super() this.kafka = new Kafka({ clientId: `plugin-server-test-${new UUIDT()}`, - brokers: process.env.KAFKA_HOSTS!.split(','), + brokers: (process.env.KAFKA_HOSTS || '').split(','), logLevel: logLevel.NOTHING, }) this.producer = this.kafka.producer() diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 798ae87d..4e2842c5 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -144,6 +144,7 @@ describe('process event', () => { expect(event.distinct_id).toEqual('2') expect(distinctIds).toEqual(['2']) expect(event.event).toEqual('$autocapture') + expect(event.elements_hash).toEqual('0679137c0cd2408a2906839143e7a71f') const elements = await getElements(event) expect(elements[0].tag_name).toEqual('a') @@ -527,7 +528,7 @@ describe('process event', () => { }) }) - test.skip('long htext', async () => { + test('long htext', async () => { await eventsProcessor.processEvent( 'new_distinct_id', '', @@ -556,6 +557,7 @@ describe('process event', () => { ) const [event] = await getEvents() + expect(event.elements_hash).toEqual('c2659b28e72835706835764cf7f63c2a') const [element] = await getElements(event) expect(element.href?.length).toEqual(2048) expect(element.text?.length).toEqual(400) @@ -594,6 +596,9 @@ describe('process event', () => { team = await getFirstTeam() expect(team.ingested_event).toEqual(true) + + const [event] = await getEvents() + expect(event.elements_hash).toEqual('a89021a60b3497d24e93ae181fba01aa') }) test('snapshot event stored as session_recording_event', async () => { From c2a7296ea4c9a1763c2b3c2e2f2bfa954a9859bd Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 13:27:30 +0100 Subject: [PATCH 29/47] create elements and element groups --- src/ingestion/process-event.ts | 37 +++++++++++++-- src/ingestion/utils.ts | 69 +++++++++++++--------------- src/types.ts | 6 +++ tests/helpers/sql.ts | 2 + tests/postgres/process-event.test.ts | 5 +- 5 files changed, 75 insertions(+), 44 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 1f83477a..b975bf98 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -3,6 +3,7 @@ import { DateTime, Duration } from 'luxon' import { CohortPeople, Element, + ElementGroup, Person, PersonDistinctId, PluginsServer, @@ -456,7 +457,7 @@ export class EventsProcessor { } else { let elementsHash = '' if (elements && elements.length > 0) { - elementsHash = await this.createElementGroup(elements) + elementsHash = await this.createElementGroup(elements, team.id) } const insertResult = await this.db.postgresQuery( 'INSERT INTO posthog_event (created_at, event, distinct_id, properties, team_id, timestamp, elements, elements_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *', @@ -489,11 +490,41 @@ export class EventsProcessor { return data } - private async createElementGroup(elements: Element[]): Promise { + private async createElementGroup(elements: Element[], teamId: number): Promise { const cleanedElements = elements.map((element, index) => ({ ...element, order: index })) const hash = hashElements(cleanedElements) - // TODO: create actual element group and elements + try { + const insertResult = await this.db.postgresQuery( + 'INSERT INTO posthog_elementgroup (hash, team_id) VALUES ($1, $2) RETURNING *', + [hash, teamId] + ) + const elementGroup = insertResult.rows[0] as ElementGroup + for (const element of cleanedElements) { + await this.db.postgresQuery( + 'INSERT INTO posthog_element (text, tag_name, href, attr_id, nth_child, nth_of_type, attributes, "order", event_id, attr_class, group_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)', + [ + element.text, + element.tag_name, + element.href, + element.attr_id, + element.nth_child, + element.nth_of_type, + element.attributes, + element.order, + element.event_id, + element.attr_class, + elementGroup.id, + ] + ) + } + } catch (error) { + // Throw further if not postgres error nr "23505" == "unique_violation" + // https://www.postgresql.org/docs/12/errcodes-appendix.html + if (error.code !== '23505') { + throw error + } + } return hash } diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index d8dd44e3..409b6b0d 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -69,48 +69,41 @@ export function sanitizeEventName(eventName: any): string { return eventName.substr(0, 200) } -export function hashElements(elements: Element[]): string { - const elementsList = elements - .map((element) => ({ - attributes: element.attributes ?? null, - text: element.text ?? null, - tag_name: element.tag_name ?? null, - href: element.href ?? null, - attr_id: element.attr_id ?? null, - attr_class: element.attr_class ?? null, - nth_child: element.nth_child ?? null, - nth_of_type: element.nth_of_type ?? null, - order: element.order ?? null, - })) - .map((element) => { - const newElement: Record = {} - for (const key of Object.keys(element).sort()) { - newElement[key] = element[key as keyof typeof element] - } - return newElement - }) - - // escape utf-8 characters into `\u1234` - function jsonEscapeUTF(s: string): string { - return s.replace(/[^\x20-\x7F]/g, (x) => '\\u' + ('000' + x.codePointAt(0)?.toString(16)).slice(-4)) - } +// escape utf-8 characters into `\u1234` +function jsonEscapeUTF8(s: string): string { + return s.replace(/[^\x20-\x7F]/g, (x) => '\\u' + ('000' + x.codePointAt(0)?.toString(16)).slice(-4)) +} - // produce output compatible to that of python's json.dumps - function pythonDumps(obj: any): string { - if (typeof obj === 'object' && obj !== null) { - if (Array.isArray(obj)) { - return `[${obj.map(pythonDumps).join(', ')}]` - } else { - return `{${Object.entries(obj) - .map(([k, v]) => `${JSON.stringify(k)}: ${pythonDumps(v)}`) - .join(', ')}}` - } - } else if (typeof obj === 'string') { - return jsonEscapeUTF(JSON.stringify(obj)) +// produce output compatible to that of python's json.dumps +function pythonDumps(obj: any): string { + if (typeof obj === 'object' && obj !== null) { + if (Array.isArray(obj)) { + return `[${obj.map(pythonDumps).join(', ')}]` // space after comma } else { - return JSON.stringify(obj) + return `{${Object.keys(obj) // no space after '{' or before '}' + .sort() // must sort the keys of the object! + .map((k) => `${pythonDumps(k)}: ${pythonDumps(obj[k])}`) // space after ':' + .join(', ')}}` // space after ',' } + } else if (typeof obj === 'string') { + return jsonEscapeUTF8(JSON.stringify(obj)) + } else { + return JSON.stringify(obj) } +} + +export function hashElements(elements: Element[]): string { + const elementsList = elements.map((element) => ({ + attributes: element.attributes ?? null, + text: element.text ?? null, + tag_name: element.tag_name ?? null, + href: element.href ?? null, + attr_id: element.attr_id ?? null, + attr_class: element.attr_class ?? null, + nth_child: element.nth_child ?? null, + nth_of_type: element.nth_of_type ?? null, + order: element.order ?? null, + })) const serializedString = pythonDumps(elementsList) diff --git a/src/types.ts b/src/types.ts index 48c1aa41..5fbf55f1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -242,6 +242,12 @@ export interface Element { group_id?: number } +export interface ElementGroup { + id: number + hash: string + team_id: number +} + /** Usable Event model. */ export interface Event { id: number diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 9b7c2ca2..4d6285a4 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -6,6 +6,8 @@ import { delay, UUIDT } from '../../src/utils' export async function resetTestDatabase(code: string): Promise { const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) const mocks = makePluginObjects(code) + await db.query('DELETE FROM posthog_element') + await db.query('DELETE FROM posthog_elementgroup') await db.query('DELETE FROM posthog_sessionrecordingevent') await db.query('DELETE FROM posthog_persondistinctid') await db.query('DELETE FROM posthog_person') diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 4e2842c5..c57540e0 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,4 +1,4 @@ -import { PluginEvent, Properties } from '@posthog/plugin-scaffold/src/types' +import { PluginEvent } from '@posthog/plugin-scaffold/src/types' import { createServer } from '../../src/server' import { LogLevel, @@ -8,7 +8,6 @@ import { Person, PersonDistinctId, Element, - SessionRecordingEvent, PostgresSessionRecordingEvent, } from '../../src/types' import { resetTestDatabase } from '../helpers/sql' @@ -64,7 +63,7 @@ async function getFirstTeam(): Promise { } async function getElements(event: Event): Promise { - return [] + return (await server.db.postgresQuery('SELECT * FROM posthog_element')).rows } async function createPerson(team: Team, distinctIds: string[], properties: Record = {}) { From 8f429cdff7ed411f9fb05b8cfbbf5a35620adc50 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 13:32:34 +0100 Subject: [PATCH 30/47] "key in object" only works with objects, not arrays --- src/ingestion/process-event.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index b975bf98..adba146d 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -378,13 +378,13 @@ export class EventsProcessor { team.ingested_event = true save = true } - if (team.event_names && !(event in team.event_names)) { + if (team.event_names && !team.event_names.includes(event)) { save = true team.event_names.push(event) team.event_names_with_usage.push({ event: event, usage_count: null, volume: null }) } for (const [key, value] of Object.entries(properties)) { - if (team.event_properties && !(key in team.event_properties)) { + if (team.event_properties && !team.event_properties.includes(key)) { team.event_properties.push(key) team.event_properties_with_usage.push({ key: key, usage_count: null, volume: null }) save = true @@ -392,7 +392,7 @@ export class EventsProcessor { if ( typeof value === 'number' && team.event_properties_numerical && - !(key in team.event_properties_numerical) + !team.event_properties_numerical.includes(key) ) { team.event_properties_numerical.push(key) save = true From d44ace81166446bf841e55ca51a27c863bac673b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 13:33:12 +0100 Subject: [PATCH 31/47] test an extra thing --- tests/postgres/process-event.test.ts | 31 ++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index c57540e0..7d0fc409 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -136,8 +136,35 @@ describe('process event', () => { // num_queries += 1 // with self.assertNumQueries(num_queries): - const [event] = await getEvents() - const [person] = await getPersons() + // capture a second time to verify e.g. event_names is not ['$autocapture', '$autocapture'] + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const events = await getEvents() + const persons = await getPersons() + expect(events.length).toEqual(2) + expect(persons.length).toEqual(1) + + const [event] = events + const [person] = persons const distinctIds = await getDistinctIds(person) expect(event.distinct_id).toEqual('2') From a5011ed6dd0a422ab31111a3f63d326c32dc2c93 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 13:47:26 +0100 Subject: [PATCH 32/47] add missing awaits that caused things to be done out of order --- src/ingestion/process-event.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index adba146d..3f767092 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -67,7 +67,7 @@ export class EventsProcessor { const personUuid = new UUIDT().toString() const ts = this.handleTimestamp(data, now, sentAt) - this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) + await this.handleIdentifyOrAlias(data['event'], properties, distinctId, teamId) let result: IEvent | SessionRecordingEvent @@ -135,9 +135,14 @@ export class EventsProcessor { await this.alias(properties['$anon_distinct_id'], distinctId, teamId) } if (properties['$set'] || properties['$set_once']) { - this.updatePersonProperties(teamId, distinctId, properties['$set'] || {}, properties['$set_once'] || {}) + await this.updatePersonProperties( + teamId, + distinctId, + properties['$set'] || {}, + properties['$set_once'] || {} + ) } - this.setIsIdentified(teamId, distinctId) + await this.setIsIdentified(teamId, distinctId) } } @@ -153,7 +158,7 @@ export class EventsProcessor { true, new UUIDT().toString() ) - this.db.addDistinctId(personCreated, distinctId) + await this.db.addDistinctId(personCreated, distinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person From ead78829aa8029277bdcbbe705b973b5678ab058 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 13:47:36 +0100 Subject: [PATCH 33/47] few extra tests --- tests/postgres/process-event.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 7d0fc409..9da2340f 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -419,10 +419,12 @@ describe('process event', () => { new UUIDT().toString() ) + expect((await getPersons()).length).toBe(1) expect((await getEvents()).length).toBe(1) expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) await createPerson(team, ['old_distinct_id_2']) + expect((await getPersons()).length).toBe(2) await eventsProcessor.processEvent( 'new_distinct_id', @@ -438,6 +440,7 @@ describe('process event', () => { new UUIDT().toString() ) expect((await getEvents()).length).toBe(2) + expect((await getPersons()).length).toBe(1) expect(await getDistinctIds((await getPersons())[0])).toEqual([ 'old_distinct_id', 'new_distinct_id', From 8a99133d77790de30478446651970928970f734a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 15:47:09 +0100 Subject: [PATCH 34/47] another test --- tests/helpers/sql.ts | 105 +++++++++++++++------------ tests/postgres/process-event.test.ts | 46 +++++++++++- 2 files changed, 99 insertions(+), 52 deletions(-) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 4d6285a4..139b4485 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -22,54 +22,8 @@ export async function resetTestDatabase(code: string): Promise { await db.query('DELETE FROM posthog_user') const teamIds = mocks.pluginConfigRows.map((c) => c.team_id) - await insertRow(db, 'posthog_user', { - id: commonUserId, - password: 'gibberish', - first_name: 'PluginTest', - last_name: 'User', - email: 'test@posthog.com', - distinct_id: 'plugin_test_user_distinct_id', - is_staff: false, - is_active: false, - date_joined: new Date().toISOString(), - }) - await insertRow(db, 'posthog_organization', { - id: commonOrganizationId, - name: 'TEST ORG', - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - }) - await insertRow(db, 'posthog_organizationmembership', { - id: commonOrganizationMembershipId, - organization_id: commonOrganizationId, - user_id: commonUserId, - level: 15, - joined_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - }) - for (const teamId of teamIds) { - await insertRow(db, 'posthog_team', { - id: teamId, - organization_id: commonOrganizationId, - app_urls: [], - name: 'TEST PROJECT', - event_names: JSON.stringify([]), - event_names_with_usage: JSON.stringify([]), - event_properties: JSON.stringify([]), - event_properties_with_usage: JSON.stringify([]), - event_properties_numerical: JSON.stringify([]), - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - anonymize_ips: false, - completed_snippet_onboarding: true, - ingested_event: true, - uuid: new UUIDT().toString(), - session_recording_opt_in: true, - plugins_opt_in: true, - opt_out_capture: false, - is_demo: false, - }) - } + await createUserTeamAndOrganization(db, teamIds[0]) + for (const plugin of mocks.pluginRows) { await insertRow(db, 'posthog_plugin', plugin) } @@ -97,3 +51,58 @@ async function insertRow(db: Pool, table: string, object: Record): throw error } } + +export async function createUserTeamAndOrganization( + db: Pool, + teamId: number, + userId: number = commonUserId, + organizationId: string = commonOrganizationId, + organizationMembershipId: string = commonOrganizationMembershipId +) { + await insertRow(db, 'posthog_user', { + id: userId, + password: 'gibberish', + first_name: 'PluginTest', + last_name: 'User', + email: `test${userId}@posthog.com`, + distinct_id: `plugin_test_user_distinct_id_${userId}`, + is_staff: false, + is_active: false, + date_joined: new Date().toISOString(), + }) + await insertRow(db, 'posthog_organization', { + id: organizationId, + name: 'TEST ORG', + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }) + await insertRow(db, 'posthog_organizationmembership', { + id: organizationMembershipId, + organization_id: organizationId, + user_id: userId, + level: 15, + joined_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }) + await insertRow(db, 'posthog_team', { + id: teamId, + organization_id: organizationId, + app_urls: [], + name: 'TEST PROJECT', + event_names: JSON.stringify([]), + event_names_with_usage: JSON.stringify([]), + event_properties: JSON.stringify([]), + event_properties_with_usage: JSON.stringify([]), + event_properties_numerical: JSON.stringify([]), + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + anonymize_ips: false, + completed_snippet_onboarding: true, + ingested_event: true, + uuid: new UUIDT().toString(), + session_recording_opt_in: true, + plugins_opt_in: true, + opt_out_capture: false, + is_demo: false, + }) +} diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 9da2340f..525bf7e7 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -10,7 +10,7 @@ import { Element, PostgresSessionRecordingEvent, } from '../../src/types' -import { resetTestDatabase } from '../helpers/sql' +import { createUserTeamAndOrganization, resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' import { UUIDT } from '../../src/utils' @@ -58,8 +58,11 @@ async function getDistinctIds(person: Person) { return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) } +async function getTeams(): Promise { + return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows +} async function getFirstTeam(): Promise { - return (await server.db.postgresQuery('SELECT * FROM posthog_team LIMIT 1')).rows[0] + return (await getTeams())[0] } async function getElements(event: Event): Promise { @@ -618,7 +621,7 @@ describe('process event', () => { new UUIDT().toString() ) - expect(eventsProcessor.posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id') + expect(eventsProcessor.posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id_1001') expect(eventsProcessor.posthog.capture).toHaveBeenCalledWith('first team event ingested', { team: team.uuid, }) @@ -899,7 +902,42 @@ describe('process event', () => { }) test('distinct team leakage', async () => { - expect(true).toBe(false) + await createUserTeamAndOrganization( + server.postgres, + 3, + 1002, + '01774e2f-0d01-0000-ee94-9a238640c6ee', + '0174f81e-36f5-0000-7ef8-cc26c1fbab1c' + ) + const team2 = (await getTeams())[1] + await createPerson(team2, ['2'], { email: 'team2@gmail.com' }) + await createPerson(team, ['1', '2']) + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: '1', + token: team.api_token, + distinct_id: '2', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const people = await getPersons() + expect(people.length).toEqual(2) + expect(people[1].team_id).toEqual(team.id) + expect(people[1].properties).toEqual({}) + expect(await getDistinctIds(people[1])).toEqual(['1', '2']) + expect(people[0].team_id).toEqual(team2.id) + expect(await getDistinctIds(people[0])).toEqual(['2']) }) test('set is_identified', async () => { From f0d14e2000c4f85265af871ad67664ac2227fd2f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 16:00:17 +0100 Subject: [PATCH 35/47] await for things to happen --- src/ingestion/process-event.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 3f767092..7d25c6f6 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -209,13 +209,13 @@ export class EventsProcessor { if (oldPerson && !newPerson) { try { - this.db.addDistinctId(oldPerson, distinctId) + await this.db.addDistinctId(oldPerson, distinctId) // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error if (retryIfFailed) { // run everything again to merge the users if needed - this.alias(previousDistinctId, distinctId, teamId, false) + await this.alias(previousDistinctId, distinctId, teamId, false) } } return @@ -223,13 +223,13 @@ export class EventsProcessor { if (!oldPerson && newPerson) { try { - this.db.addDistinctId(newPerson, previousDistinctId) + await this.db.addDistinctId(newPerson, previousDistinctId) // Catch race case when somebody already added this distinct_id between .get and .addDistinctId } catch { // integrity error if (retryIfFailed) { // run everything again to merge the users if needed - this.alias(previousDistinctId, distinctId, teamId, false) + await this.alias(previousDistinctId, distinctId, teamId, false) } } return @@ -245,21 +245,21 @@ export class EventsProcessor { false, new UUIDT().toString() ) - this.db.addDistinctId(personCreated, distinctId) - this.db.addDistinctId(personCreated, previousDistinctId) + await this.db.addDistinctId(personCreated, distinctId) + await this.db.addDistinctId(personCreated, previousDistinctId) } catch { // Catch race condition where in between getting and creating, // another request already created this person if (retryIfFailed) { // Try once more, probably one of the two persons exists now - this.alias(previousDistinctId, distinctId, teamId, false) + await this.alias(previousDistinctId, distinctId, teamId, false) } } return } if (oldPerson && newPerson && oldPerson.id !== newPerson.id) { - this.mergePeople(newPerson, [oldPerson]) + await this.mergePeople(newPerson, [oldPerson]) } } @@ -340,7 +340,7 @@ export class EventsProcessor { properties['$ip'] = ip } - this.storeNamesAndProperties(team, event, properties) + await this.storeNamesAndProperties(team, event, properties) const pdiSelectResult = await this.db.postgresQuery( 'SELECT COUNT(*) AS pdicount FROM posthog_persondistinctid WHERE team_id = $1 AND distinct_id = $2', From 684558f012c80cc50ea2e813005f28279a6c046f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 16:06:36 +0100 Subject: [PATCH 36/47] fix to work with latest master --- tests/helpers/sql.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 139b4485..0529111f 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -75,6 +75,7 @@ export async function createUserTeamAndOrganization( name: 'TEST ORG', created_at: new Date().toISOString(), updated_at: new Date().toISOString(), + personalization: '{}', }) await insertRow(db, 'posthog_organizationmembership', { id: organizationMembershipId, From 14dff2bf41d0ca1f7655d4420c7c52b007ab2aa1 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 17:26:49 +0100 Subject: [PATCH 37/47] client is called twice - it's initialized for sending celery tasks on webhooks as well --- tests/postgres/vm.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index 7a65b9ee..cff117ed 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -712,7 +712,7 @@ test('posthog in runEvery', async () => { const response = await vm.tasks.runEveryMinute.exec() expect(response).toBe('haha') - expect(Client).toHaveBeenCalledTimes(1) + expect(Client).toHaveBeenCalledTimes(2) expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) const mockClientInstance = (Client as any).mock.instances[0] @@ -750,7 +750,7 @@ test('posthog in runEvery with timestamp', async () => { const response = await vm.tasks.runEveryMinute.exec() expect(response).toBe('haha') - expect(Client).toHaveBeenCalledTimes(1) + expect(Client).toHaveBeenCalledTimes(2) expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) const mockClientInstance = (Client as any).mock.instances[0] From 61799ac781390432bc3349e95821928bab020558 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 29 Jan 2021 18:03:20 +0100 Subject: [PATCH 38/47] check webhook celery client queue --- src/ingestion/process-event.ts | 2 +- tests/postgres/vm.test.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 7d25c6f6..1f0aa443 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -37,7 +37,7 @@ export class EventsProcessor { this.db = pluginsServer.db this.clickhouse = pluginsServer.clickhouse! this.kafkaProducer = pluginsServer.kafkaProducer! - this.celery = new Client(pluginsServer.redis) + this.celery = new Client(pluginsServer.redis, pluginsServer.CELERY_DEFAULT_QUEUE) this.posthog = nodePostHog('sTMFPsFhdP1Ssg') if (process.env.NODE_ENV === 'test') { this.posthog.optOut() diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index cff117ed..77245030 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -713,7 +713,8 @@ test('posthog in runEvery', async () => { expect(response).toBe('haha') expect(Client).toHaveBeenCalledTimes(2) - expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) + expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue const mockClientInstance = (Client as any).mock.instances[0] const mockSendTask = mockClientInstance.sendTask @@ -751,7 +752,8 @@ test('posthog in runEvery with timestamp', async () => { expect(response).toBe('haha') expect(Client).toHaveBeenCalledTimes(2) - expect((Client as any).mock.calls[0][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) + expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue + expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue const mockClientInstance = (Client as any).mock.instances[0] const mockSendTask = mockClientInstance.sendTask From 9276db92cef7dd5152fcd2546b77e36a286324c4 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 09:02:45 +0100 Subject: [PATCH 39/47] split into postgres & shared process event test --- package.json | 3 +- src/db.ts | 12 +- tests/helpers/sql.ts | 2 +- tests/postgres/process-event.test.ts | 1010 +------------------------ tests/shared/process-event.test.ts | 1024 ++++++++++++++++++++++++++ 5 files changed, 1063 insertions(+), 988 deletions(-) create mode 100644 tests/shared/process-event.test.ts diff --git a/package.json b/package.json index a76030d2..d6eee6a1 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "prettier": "prettier --write .", "prettier:check": "prettier --check .", "prepare": "yarn compile:protobuf", - "prepublishOnly": "yarn build" + "prepublishOnly": "yarn build", + "setup:dev": "cd ../posthog && dropdb test_posthog && createdb test_posthog && source env/bin/activate && DATABASE_URL=postgres://localhost:5432/test_posthog DEBUG=1 python manage.py migrate" }, "bin": { "posthog-plugin-server": "bin/posthog-plugin-server" diff --git a/src/db.ts b/src/db.ts index 2ca8821d..95256404 100644 --- a/src/db.ts +++ b/src/db.ts @@ -5,8 +5,8 @@ import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { unparsePersonPartial } from './ingestion/utils' -import { Person, PersonDistinctId, RawPerson, RawOrganization } from './types' -import { castTimestampOrNow, sanitizeSqlIdentifier } from './utils' +import { Person, PersonDistinctId, RawPerson, RawOrganization, Team } from './types' +import { castTimestampOrNow, sanitizeSqlIdentifier, UUIDT } from './utils' /** The recommended way of accessing the database. */ export class DB { @@ -57,7 +57,8 @@ export class DB { teamId: number, isUserId: number | null, isIdentified: boolean, - uuid: string + uuid: string, + distinctIds?: string[] ): Promise { const insertResult = await this.postgresQuery( 'INSERT INTO posthog_person (created_at, properties, team_id, is_user_id, is_identified, uuid) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *', @@ -77,6 +78,11 @@ export class DB { messages: [{ value: Buffer.from(JSON.stringify(data)) }], }) } + + for (const distinctId of distinctIds || []) { + await this.addDistinctId(personCreated, distinctId) + } + return personCreated } diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 0529111f..75aac767 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -58,7 +58,7 @@ export async function createUserTeamAndOrganization( userId: number = commonUserId, organizationId: string = commonOrganizationId, organizationMembershipId: string = commonOrganizationMembershipId -) { +): Promise { await insertRow(db, 'posthog_user', { id: userId, password: 'gibberish', diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 525bf7e7..c9c2513f 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,7 +1,4 @@ -import { PluginEvent } from '@posthog/plugin-scaffold/src/types' -import { createServer } from '../../src/server' import { - LogLevel, PluginsServer, Team, Event, @@ -10,47 +7,28 @@ import { Element, PostgresSessionRecordingEvent, } from '../../src/types' -import { createUserTeamAndOrganization, resetTestDatabase } from '../helpers/sql' -import { EventsProcessor } from '../../src/ingestion/process-event' import { DateTime } from 'luxon' import { UUIDT } from '../../src/utils' +import { createProcessEventTests } from '../shared/process-event.test' jest.setTimeout(600000) // 600 sec timeout -let team: Team -let server: PluginsServer -let stopServer: () => Promise -let eventsProcessor: EventsProcessor -let now = DateTime.utc() - -async function getServer(): Promise<[PluginsServer, () => Promise]> { - const [server, stopServer] = await createServer({ - PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', - CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', - LOG_LEVEL: LogLevel.Log, - }) - - await server.redis.del(server.PLUGINS_CELERY_QUEUE) - await server.redis.del(server.CELERY_DEFAULT_QUEUE) - return [server, stopServer] -} - -async function getSessionRecordingEvents(): Promise { +async function getSessionRecordingEvents(server: PluginsServer): Promise { const result = await server.db.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') return result.rows as PostgresSessionRecordingEvent[] } -async function getEvents(): Promise { +async function getEvents(server: PluginsServer): Promise { const result = await server.db.postgresQuery('SELECT * FROM posthog_event') return result.rows as Event[] } -async function getPersons(): Promise { +async function getPersons(server: PluginsServer): Promise { const result = await server.db.postgresQuery('SELECT * FROM posthog_person') return result.rows as Person[] } -async function getDistinctIds(person: Person) { +async function getDistinctIds(server: PluginsServer, person: Person): Promise { const result = await server.db.postgresQuery( 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', [person.id, person.team_id] @@ -58,970 +36,36 @@ async function getDistinctIds(person: Person) { return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) } -async function getTeams(): Promise { +async function getTeams(server: PluginsServer): Promise { return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows } -async function getFirstTeam(): Promise { - return (await getTeams())[0] + +async function getFirstTeam(server: PluginsServer): Promise { + return (await getTeams(server))[0] } -async function getElements(event: Event): Promise { +async function getElements(server: PluginsServer, event: Event): Promise { return (await server.db.postgresQuery('SELECT * FROM posthog_element')).rows } -async function createPerson(team: Team, distinctIds: string[], properties: Record = {}) { - const person = await server.db.createPerson( - DateTime.utc(), - properties, - team.id, - null, - false, - new UUIDT().toString() - ) - for (const distinctId of distinctIds) { - await server.db.addDistinctId(person, distinctId) - } - - return person +async function createPerson( + server: PluginsServer, + team: Team, + distinctIds: string[], + properties: Record = {} +): Promise { + return server.db.createPerson(DateTime.utc(), properties, team.id, null, false, new UUIDT().toString(), distinctIds) } -describe('process event', () => { - beforeEach(async () => { - const testCode = ` - function processEvent (event, meta) { - event.properties["somewhere"] = "over the rainbow"; - return event - } - ` - await resetTestDatabase(testCode) - ;[server, stopServer] = await getServer() - eventsProcessor = new EventsProcessor(server) - team = await getFirstTeam() - now = DateTime.utc() - }) - - afterEach(async () => { - await stopServer?.() - }) - - test('capture new person', async () => { - await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [true, team.id]) - team = await getFirstTeam() - - expect(team.event_names).toEqual([]) - - await eventsProcessor.processEvent( - '2', - '', - '', - ({ - event: '$autocapture', - properties: { - distinct_id: 2, - token: team.api_token, - $elements: [ - { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, - { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, - ], - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - // TODO: add this back? - // num_queries = 28 - // if settings.EE_AVAILABLE: # extra queries to check for hooks - // num_queries += 4 - // if settings.MULTI_TENANCY: # extra query to check for billing plan - // num_queries += 1 - // with self.assertNumQueries(num_queries): - - // capture a second time to verify e.g. event_names is not ['$autocapture', '$autocapture'] - await eventsProcessor.processEvent( - '2', - '', - '', - ({ - event: '$autocapture', - properties: { - distinct_id: 2, - token: team.api_token, - $elements: [ - { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, - { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, - ], - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const events = await getEvents() - const persons = await getPersons() - expect(events.length).toEqual(2) - expect(persons.length).toEqual(1) - - const [event] = events - const [person] = persons - const distinctIds = await getDistinctIds(person) - - expect(event.distinct_id).toEqual('2') - expect(distinctIds).toEqual(['2']) - expect(event.event).toEqual('$autocapture') - expect(event.elements_hash).toEqual('0679137c0cd2408a2906839143e7a71f') - - const elements = await getElements(event) - expect(elements[0].tag_name).toEqual('a') - expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) - expect(elements[1].order).toEqual(1) - expect(elements[1].text).toEqual('💻') - - team = await getFirstTeam() - expect(team.event_names).toEqual(['$autocapture']) - expect(team.event_names_with_usage).toEqual([{ event: '$autocapture', volume: null, usage_count: null }]) - expect(team.event_properties).toEqual(['distinct_id', 'token', '$ip']) - expect(team.event_properties_with_usage).toEqual([ - { key: 'distinct_id', usage_count: null, volume: null }, - { key: 'token', usage_count: null, volume: null }, - { key: '$ip', usage_count: null, volume: null }, - ]) - }) - - test('capture no element', async () => { - await createPerson(team, ['asdfasdfasdf']) - - await eventsProcessor.processEvent( - 'asdfasdfasdf', - '', - '', - ({ - event: '$pageview', - properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect(await getDistinctIds((await getPersons())[0])).toEqual(['asdfasdfasdf']) - const [event] = await getEvents() - expect(event.event).toBe('$pageview') - }) - - test('capture sent_at', async () => { - await createPerson(team, ['asdfasdfasdf']) - - const rightNow = DateTime.utc() - const tomorrow = rightNow.plus({ days: 1, hours: 2 }) - const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }) - - await eventsProcessor.processEvent( - 'movie played', - '', - '', - ({ - event: '$pageview', - timestamp: tomorrow.toISO(), - properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - rightNow, - tomorrowSentAt, - new UUIDT().toString() - ) - - const [event] = await getEvents() - const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds - - expect(eventSecondsBeforeNow).toBeGreaterThan(590) - expect(eventSecondsBeforeNow).toBeLessThan(610) - }) - - test('capture sent_at no timezones', async () => { - await createPerson(team, ['asdfasdfasdf']) - - const rightNow = DateTime.utc() - const tomorrow = rightNow.plus({ days: 1, hours: 2 }).setZone('UTC+4') - const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }).setZone('UTC+4') - - // TODO: not sure if this is correct? - // tomorrow = tomorrow.replace(tzinfo=None) - // tomorrow_sent_at = tomorrow_sent_at.replace(tzinfo=None) - - await eventsProcessor.processEvent( - 'movie played', - '', - '', - ({ - event: '$pageview', - timestamp: tomorrow, - properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - rightNow, - tomorrowSentAt, - new UUIDT().toString() - ) - - const [event] = await getEvents() - const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds - - expect(eventSecondsBeforeNow).toBeGreaterThan(590) - expect(eventSecondsBeforeNow).toBeLessThan(610) - }) - - test('capture no sent_at', async () => { - await createPerson(team, ['asdfasdfasdf']) - - const rightNow = DateTime.utc() - const tomorrow = rightNow.plus({ days: 1, hours: 2 }) - - await eventsProcessor.processEvent( - 'movie played', - '', - '', - ({ - event: '$pageview', - timestamp: tomorrow.toISO(), - properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - rightNow, - null, - new UUIDT().toString() - ) - - const [event] = await getEvents() - const difference = tomorrow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds - expect(difference).toBeLessThan(1) - }) - - test('ip capture', async () => { - await createPerson(team, ['asdfasdfasdf']) - - await eventsProcessor.processEvent( - 'asdfasdfasdf', - '11.12.13.14', - '', - ({ - event: '$pageview', - properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - const [event] = await getEvents() - expect(event.properties['$ip']).toBe('11.12.13.14') - }) - - test('ip override', async () => { - await createPerson(team, ['asdfasdfasdf']) - - await eventsProcessor.processEvent( - 'asdfasdfasdf', - '11.12.13.14', - '', - ({ - event: '$pageview', - properties: { $ip: '1.0.0.1', distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - const [event] = await getEvents() - expect(event.properties['$ip']).toBe('1.0.0.1') - }) - - test('anonymized ip capture', async () => { - await server.db.postgresQuery('update posthog_team set anonymize_ips = $1', [true]) - await createPerson(team, ['asdfasdfasdf']) - - await eventsProcessor.processEvent( - 'asdfasdfasdf', - '11.12.13.14', - '', - ({ - event: '$pageview', - properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - const [event] = await getEvents() - expect(event.properties['$ip']).not.toBeDefined() - }) - - test('alias', async () => { - await createPerson(team, ['old_distinct_id']) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$create_alias', - properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) - }) - - test('alias reverse', async () => { - await createPerson(team, ['old_distinct_id']) - - await eventsProcessor.processEvent( - 'old_distinct_id', - '', - '', - ({ - event: '$create_alias', - properties: { distinct_id: 'old_distinct_id', token: team.api_token, alias: 'new_distinct_id' }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) - }) - - test('alias twice', async () => { - await createPerson(team, ['old_distinct_id']) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$create_alias', - properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getPersons()).length).toBe(1) - expect((await getEvents()).length).toBe(1) - expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) - - await createPerson(team, ['old_distinct_id_2']) - expect((await getPersons()).length).toBe(2) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$create_alias', - properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id_2' }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - expect((await getEvents()).length).toBe(2) - expect((await getPersons()).length).toBe(1) - expect(await getDistinctIds((await getPersons())[0])).toEqual([ - 'old_distinct_id', - 'new_distinct_id', - 'old_distinct_id_2', - ]) - }) - - test('alias before person', async () => { - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$create_alias', - properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - expect((await getPersons()).length).toBe(1) - expect(await getDistinctIds((await getPersons())[0])).toEqual(['new_distinct_id', 'old_distinct_id']) - }) - - test('alias both existing', async () => { - await createPerson(team, ['old_distinct_id']) - await createPerson(team, ['new_distinct_id']) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$create_alias', - properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - expect(await getDistinctIds((await getPersons())[0])).toEqual(['old_distinct_id', 'new_distinct_id']) - }) - - test('offset timestamp', async () => { - now = DateTime.fromISO('2020-01-01T12:00:05.200Z') - - await eventsProcessor.processEvent( - 'distinct_id', - '', - '', - ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - expect((await getEvents()).length).toBe(1) - - const [event] = await getEvents() - expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') - }) - - test('offset timestamp no sent_at', async () => { - now = DateTime.fromISO('2020-01-01T12:00:05.200Z') - - await eventsProcessor.processEvent( - 'distinct_id', - '', - '', - ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, - team.id, - now, - null, - new UUIDT().toString() - ) - expect((await getEvents()).length).toBe(1) - - const [event] = await getEvents() - expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') - }) - - test('alias merge properties', async () => { - await createPerson(team, ['old_distinct_id'], { key_on_both: 'old value both', key_on_old: 'old value' }) - await createPerson(team, ['new_distinct_id'], { key_on_both: 'new value both', key_on_new: 'new value' }) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$create_alias', - properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - expect((await getPersons()).length).toBe(1) - const [person] = await getPersons() - expect(await getDistinctIds(person)).toEqual(['old_distinct_id', 'new_distinct_id']) - expect(person.properties).toEqual({ - key_on_both: 'new value both', - key_on_new: 'new value', - key_on_old: 'old value', - }) - }) - - test('long htext', async () => { - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$autocapture', - properties: { - distinct_id: 'new_distinct_id', - token: team.api_token, - $elements: [ - { - tag_name: 'a', - $el_text: 'a'.repeat(2050), - attr__href: 'a'.repeat(2050), - nth_child: 1, - nth_of_type: 2, - attr__class: 'btn btn-sm', - }, - ], - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const [event] = await getEvents() - expect(event.elements_hash).toEqual('c2659b28e72835706835764cf7f63c2a') - const [element] = await getElements(event) - expect(element.href?.length).toEqual(2048) - expect(element.text?.length).toEqual(400) - }) - - test('capture first team event', async () => { - await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [false, team.id]) - - eventsProcessor.posthog = { - identify: jest.fn((distinctId) => true), - capture: jest.fn((event, properties) => true), - } as any - - await eventsProcessor.processEvent( - '2', - '', - '', - ({ - event: '$autocapture', - properties: { - distinct_id: 1, - token: team.api_token, - $elements: [{ tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }], - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect(eventsProcessor.posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id_1001') - expect(eventsProcessor.posthog.capture).toHaveBeenCalledWith('first team event ingested', { - team: team.uuid, - }) - - team = await getFirstTeam() - expect(team.ingested_event).toEqual(true) - - const [event] = await getEvents() - expect(event.elements_hash).toEqual('a89021a60b3497d24e93ae181fba01aa') - }) - - test('snapshot event stored as session_recording_event', async () => { - await eventsProcessor.processEvent( - 'some-id', - '', - '', - ({ - event: '$snapshot', - properties: { $session_id: 'abcf-efg', $snapshot_data: { timestamp: 123 } }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const events = await getEvents() - expect(events.length).toEqual(0) - - const sessionRecordingEvents = await getSessionRecordingEvents() - expect(sessionRecordingEvents.length).toBe(1) - - const [event] = sessionRecordingEvents - expect(event.session_id).toEqual('abcf-efg') - expect(event.distinct_id).toEqual('some-id') - expect(event.snapshot_data).toEqual({ timestamp: 123 }) - }) - - test('identify set', async () => { - await createPerson(team, ['distinct_id']) - - await eventsProcessor.processEvent( - 'distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - token: team.api_token, - distinct_id: 'distinct_id', - $set: { a_prop: 'test-1', c_prop: 'test-1' }, - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - - const [event] = await getEvents() - expect(event.properties['$set']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) - - const [person] = await getPersons() - expect(await getDistinctIds(person)).toEqual(['distinct_id']) - expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) - expect(person.is_identified).toEqual(true) - - await eventsProcessor.processEvent( - 'distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - token: team.api_token, - distinct_id: 'distinct_id', - $set: { a_prop: 'test-2', b_prop: 'test-2b' }, - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - expect((await getEvents()).length).toBe(2) - const [person2] = await getPersons() - expect(person2.properties).toEqual({ a_prop: 'test-2', b_prop: 'test-2b', c_prop: 'test-1' }) - }) - - test('identify set_once', async () => { - await createPerson(team, ['distinct_id']) - - await eventsProcessor.processEvent( - 'distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - token: team.api_token, - distinct_id: 'distinct_id', - $set_once: { a_prop: 'test-1', c_prop: 'test-1' }, - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - - const [event] = await getEvents() - expect(event.properties['$set_once']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) - - const [person] = await getPersons() - expect(await getDistinctIds(person)).toEqual(['distinct_id']) - expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) - expect(person.is_identified).toEqual(true) - - await eventsProcessor.processEvent( - 'distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - token: team.api_token, - distinct_id: 'distinct_id', - $set_once: { a_prop: 'test-2', b_prop: 'test-2b' }, - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - expect((await getEvents()).length).toBe(2) - const [person2] = await getPersons() - expect(person2.properties).toEqual({ a_prop: 'test-1', b_prop: 'test-2b', c_prop: 'test-1' }) - }) - - test('distinct with anonymous_id', async () => { - await createPerson(team, ['anonymous_id']) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - $anon_distinct_id: 'anonymous_id', - token: team.api_token, - distinct_id: 'new_distinct_id', - $set: { a_prop: 'test' }, - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - expect((await getEvents()).length).toBe(1) - const [event] = await getEvents() - expect(event.properties['$set']).toEqual({ a_prop: 'test' }) - const [person] = await getPersons() - expect(await getDistinctIds(person)).toEqual(['anonymous_id', 'new_distinct_id']) - expect(person.properties).toEqual({ a_prop: 'test' }) - - // check no errors as this call can happen multiple times - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - $anon_distinct_id: 'anonymous_id', - token: team.api_token, - distinct_id: 'new_distinct_id', - $set: { a_prop: 'test' }, - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - }) - - // This case is likely to happen after signup, for example: - // 1. User browses website with anonymous_id - // 2. User signs up, triggers event with their new_distinct_id (creating a new Person) - // 3. In the frontend, try to alias anonymous_id with new_distinct_id - // Result should be that we end up with one Person with both ID's - test('distinct with anonymous_id which was already created', async () => { - await createPerson(team, ['anonymous_id']) - await createPerson(team, ['new_distinct_id'], { email: 'someone@gmail.com' }) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - $anon_distinct_id: 'anonymous_id', - token: team.api_token, - distinct_id: 'new_distinct_id', - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const [person] = await getPersons() - expect(await getDistinctIds(person)).toEqual(['anonymous_id', 'new_distinct_id']) - expect(person.properties['email']).toEqual('someone@gmail.com') - }) - - test('distinct with multiple anonymous_ids which were already created', async () => { - await createPerson(team, ['anonymous_id']) - await createPerson(team, ['new_distinct_id'], { email: 'someone@gmail.com' }) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - $anon_distinct_id: 'anonymous_id', - token: team.api_token, - distinct_id: 'new_distinct_id', - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const persons1 = await getPersons() - expect(persons1.length).toBe(1) - expect(await getDistinctIds(persons1[0])).toEqual(['anonymous_id', 'new_distinct_id']) - expect(persons1[0].properties['email']).toEqual('someone@gmail.com') - - await createPerson(team, ['anonymous_id_2']) - - await eventsProcessor.processEvent( - 'new_distinct_id', - '', - '', - ({ - event: '$identify', - properties: { - $anon_distinct_id: 'anonymous_id_2', - token: team.api_token, - distinct_id: 'new_distinct_id', - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const persons2 = await getPersons() - expect(persons2.length).toBe(1) - expect(await getDistinctIds(persons2[0])).toEqual(['anonymous_id', 'new_distinct_id', 'anonymous_id_2']) - expect(persons2[0].properties['email']).toEqual('someone@gmail.com') - }) - - test('distinct team leakage', async () => { - await createUserTeamAndOrganization( - server.postgres, - 3, - 1002, - '01774e2f-0d01-0000-ee94-9a238640c6ee', - '0174f81e-36f5-0000-7ef8-cc26c1fbab1c' - ) - const team2 = (await getTeams())[1] - await createPerson(team2, ['2'], { email: 'team2@gmail.com' }) - await createPerson(team, ['1', '2']) - - await eventsProcessor.processEvent( - '2', - '', - '', - ({ - event: '$identify', - properties: { - $anon_distinct_id: '1', - token: team.api_token, - distinct_id: '2', - }, - } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const people = await getPersons() - expect(people.length).toEqual(2) - expect(people[1].team_id).toEqual(team.id) - expect(people[1].properties).toEqual({}) - expect(await getDistinctIds(people[1])).toEqual(['1', '2']) - expect(people[0].team_id).toEqual(team2.id) - expect(await getDistinctIds(people[0])).toEqual(['2']) - }) - - test('set is_identified', async () => { - const distinct_id = '777' - const person1 = await createPerson(team, [distinct_id]) - expect(person1.is_identified).toBe(false) - - await eventsProcessor.processEvent( - distinct_id, - '', - '', - ({ event: '$identify', properties: {} } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - const [person2] = await getPersons() - expect(person2.is_identified).toBe(true) - }) - - test('team event_properties', async () => { - expect(team.event_properties_numerical).toEqual([]) - - await eventsProcessor.processEvent( - 'xxx', - '', - '', - ({ event: 'purchase', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - - team = await getFirstTeam() - expect(team.event_properties).toEqual(['price', 'name', '$ip']) - expect(team.event_properties_numerical).toEqual(['price']) - }) - - test('event name object json', async () => { - await eventsProcessor.processEvent( - 'xxx', - '', - '', - ({ event: { 'event name': 'as object' }, properties: {} } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - const [event] = await getEvents() - expect(event.event).toEqual('{"event name":"as object"}') - }) - - test('event name array json', async () => { - await eventsProcessor.processEvent( - 'xxx', - '', - '', - ({ event: ['event name', 'a list'], properties: {} } as any) as PluginEvent, - team.id, - now, - now, - new UUIDT().toString() - ) - const [event] = await getEvents() - expect(event.event).toEqual('["event name","a list"]') - }) - - test('long event name substr', async () => { - await eventsProcessor.processEvent( - 'xxx', - '', - '', - ({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, - team.id, - DateTime.utc(), - DateTime.utc(), - 'uuid' - ) - - const [event] = await getEvents() - expect(event.event?.length).toBe(200) +describe('process event (postgresql)', () => { + createProcessEventTests('postgresql', { + getSessionRecordingEvents, + getEvents, + getPersons, + getDistinctIds, + getTeams, + getFirstTeam, + getElements, + createPerson, }) }) diff --git a/tests/shared/process-event.test.ts b/tests/shared/process-event.test.ts new file mode 100644 index 00000000..f5eca0ca --- /dev/null +++ b/tests/shared/process-event.test.ts @@ -0,0 +1,1024 @@ +import { PluginEvent } from '@posthog/plugin-scaffold/src/types' +import { createServer } from '../../src/server' +import { + LogLevel, + PluginsServer, + Team, + Event, + Person, + PersonDistinctId, + Element, + PostgresSessionRecordingEvent, +} from '../../src/types' +import { createUserTeamAndOrganization, resetTestDatabase } from '../helpers/sql' +import { EventsProcessor } from '../../src/ingestion/process-event' +import { DateTime } from 'luxon' +import { UUIDT } from '../../src/utils' + +jest.setTimeout(600000) // 600 sec timeout + +export const createProcessEventTests = ( + database: 'postgresql' | 'clickhouse', + { + getSessionRecordingEvents, + getEvents, + getPersons, + getDistinctIds, + getTeams, + getFirstTeam, + getElements, + createPerson, + }: { + getSessionRecordingEvents: (server: PluginsServer) => Promise + getEvents: (server: PluginsServer) => Promise + getPersons: (server: PluginsServer) => Promise + getDistinctIds: (server: PluginsServer, person: Person) => Promise + getTeams: (server: PluginsServer) => Promise + getFirstTeam: (server: PluginsServer) => Promise + getElements: (server: PluginsServer, event: Event) => Promise + createPerson: ( + server: PluginsServer, + team: Team, + distinctIds: string[], + properties?: Record + ) => Promise + } +) => { + let team: Team + let server: PluginsServer + let stopServer: () => Promise + let eventsProcessor: EventsProcessor + let now = DateTime.utc() + + async function getServer(): Promise<[PluginsServer, () => Promise]> { + const [server, stopServer] = await createServer({ + PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', + CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', + LOG_LEVEL: LogLevel.Log, + }) + + await server.redis.del(server.PLUGINS_CELERY_QUEUE) + await server.redis.del(server.CELERY_DEFAULT_QUEUE) + return [server, stopServer] + } + + beforeEach(async () => { + const testCode = ` + function processEvent (event, meta) { + event.properties["somewhere"] = "over the rainbow"; + return event + } + ` + await resetTestDatabase(testCode) + ;[server, stopServer] = await getServer() + eventsProcessor = new EventsProcessor(server) + team = await getFirstTeam(server) + now = DateTime.utc() + }) + + afterEach(async () => { + await stopServer?.() + }) + + test('capture new person', async () => { + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [true, team.id]) + team = await getFirstTeam(server) + + expect(team.event_names).toEqual([]) + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + // TODO: add this back? + // num_queries = 28 + // if settings.EE_AVAILABLE: # extra queries to check for hooks + // num_queries += 4 + // if settings.MULTI_TENANCY: # extra query to check for billing plan + // num_queries += 1 + // with self.assertNumQueries(num_queries): + + // capture a second time to verify e.g. event_names is not ['$autocapture', '$autocapture'] + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 2, + token: team.api_token, + $elements: [ + { tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }, + { tag_name: 'div', nth_child: 1, nth_of_type: 2, $el_text: '💻' }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const events = await getEvents(server) + const persons = await getPersons(server) + expect(events.length).toEqual(2) + expect(persons.length).toEqual(1) + + const [event] = events + const [person] = persons + const distinctIds = await getDistinctIds(server, person) + + expect(event.distinct_id).toEqual('2') + expect(distinctIds).toEqual(['2']) + expect(event.event).toEqual('$autocapture') + expect(event.elements_hash).toEqual('0679137c0cd2408a2906839143e7a71f') + + const elements = await getElements(server, event) + expect(elements[0].tag_name).toEqual('a') + expect(elements[0].attr_class).toEqual(['btn', 'btn-sm']) + expect(elements[1].order).toEqual(1) + expect(elements[1].text).toEqual('💻') + + team = await getFirstTeam(server) + expect(team.event_names).toEqual(['$autocapture']) + expect(team.event_names_with_usage).toEqual([{ event: '$autocapture', volume: null, usage_count: null }]) + expect(team.event_properties).toEqual(['distinct_id', 'token', '$ip']) + expect(team.event_properties_with_usage).toEqual([ + { key: 'distinct_id', usage_count: null, volume: null }, + { key: 'token', usage_count: null, volume: null }, + { key: '$ip', usage_count: null, volume: null }, + ]) + }) + + test('capture no element', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual(['asdfasdfasdf']) + const [event] = await getEvents(server) + expect(event.event).toBe('$pageview') + }) + + test('capture sent_at', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture sent_at no timezones', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }).setZone('UTC+4') + const tomorrowSentAt = rightNow.plus({ days: 1, hours: 2, minutes: 10 }).setZone('UTC+4') + + // TODO: not sure if this is correct? + // tomorrow = tomorrow.replace(tzinfo=None) + // tomorrow_sent_at = tomorrow_sent_at.replace(tzinfo=None) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow, + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + tomorrowSentAt, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + const eventSecondsBeforeNow = rightNow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + + expect(eventSecondsBeforeNow).toBeGreaterThan(590) + expect(eventSecondsBeforeNow).toBeLessThan(610) + }) + + test('capture no sent_at', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + const rightNow = DateTime.utc() + const tomorrow = rightNow.plus({ days: 1, hours: 2 }) + + await eventsProcessor.processEvent( + 'movie played', + '', + '', + ({ + event: '$pageview', + timestamp: tomorrow.toISO(), + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + rightNow, + null, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + const difference = tomorrow.diff(DateTime.fromISO(event.timestamp), 'seconds').seconds + expect(difference).toBeLessThan(1) + }) + + test('ip capture', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.properties['$ip']).toBe('11.12.13.14') + }) + + test('ip override', async () => { + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { $ip: '1.0.0.1', distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.properties['$ip']).toBe('1.0.0.1') + }) + + test('anonymized ip capture', async () => { + await server.db.postgresQuery('update posthog_team set anonymize_ips = $1', [true]) + await createPerson(server, team, ['asdfasdfasdf']) + + await eventsProcessor.processEvent( + 'asdfasdfasdf', + '11.12.13.14', + '', + ({ + event: '$pageview', + properties: { distinct_id: 'asdfasdfasdf', token: team.api_token }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.properties['$ip']).not.toBeDefined() + }) + + test('alias', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('alias reverse', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'old_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'old_distinct_id', token: team.api_token, alias: 'new_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('alias twice', async () => { + await createPerson(server, team, ['old_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getPersons(server)).length).toBe(1) + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + + await createPerson(server, team, ['old_distinct_id_2']) + expect((await getPersons(server)).length).toBe(2) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id_2' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(2) + expect((await getPersons(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + 'old_distinct_id_2', + ]) + }) + + test('alias before person', async () => { + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect((await getPersons(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'new_distinct_id', + 'old_distinct_id', + ]) + }) + + test('alias both existing', async () => { + await createPerson(server, team, ['old_distinct_id']) + await createPerson(server, team, ['new_distinct_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect(await getDistinctIds(server, (await getPersons(server))[0])).toEqual([ + 'old_distinct_id', + 'new_distinct_id', + ]) + }) + + test('offset timestamp', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') + }) + + test('offset timestamp no sent_at', async () => { + now = DateTime.fromISO('2020-01-01T12:00:05.200Z') + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ offset: 150, event: '$autocapture', distinct_id: 'distinct_id' } as any) as PluginEvent, + team.id, + now, + null, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.timestamp).toEqual('2020-01-01T12:00:05.050Z') + }) + + test('alias merge properties', async () => { + await createPerson(server, team, ['old_distinct_id'], { + key_on_both: 'old value both', + key_on_old: 'old value', + }) + await createPerson(server, team, ['new_distinct_id'], { + key_on_both: 'new value both', + key_on_new: 'new value', + }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$create_alias', + properties: { distinct_id: 'new_distinct_id', token: team.api_token, alias: 'old_distinct_id' }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + expect((await getPersons(server)).length).toBe(1) + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['old_distinct_id', 'new_distinct_id']) + expect(person.properties).toEqual({ + key_on_both: 'new value both', + key_on_new: 'new value', + key_on_old: 'old value', + }) + }) + + test('long htext', async () => { + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 'new_distinct_id', + token: team.api_token, + $elements: [ + { + tag_name: 'a', + $el_text: 'a'.repeat(2050), + attr__href: 'a'.repeat(2050), + nth_child: 1, + nth_of_type: 2, + attr__class: 'btn btn-sm', + }, + ], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [event] = await getEvents(server) + expect(event.elements_hash).toEqual('c2659b28e72835706835764cf7f63c2a') + const [element] = await getElements(server, event) + expect(element.href?.length).toEqual(2048) + expect(element.text?.length).toEqual(400) + }) + + test('capture first team event', async () => { + await server.db.postgresQuery(`UPDATE posthog_team SET ingested_event = $1 WHERE id = $2`, [false, team.id]) + + eventsProcessor.posthog = { + identify: jest.fn((distinctId) => true), + capture: jest.fn((event, properties) => true), + } as any + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$autocapture', + properties: { + distinct_id: 1, + token: team.api_token, + $elements: [{ tag_name: 'a', nth_child: 1, nth_of_type: 2, attr__class: 'btn btn-sm' }], + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect(eventsProcessor.posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id_1001') + expect(eventsProcessor.posthog.capture).toHaveBeenCalledWith('first team event ingested', { + team: team.uuid, + }) + + team = await getFirstTeam(server) + expect(team.ingested_event).toEqual(true) + + const [event] = await getEvents(server) + expect(event.elements_hash).toEqual('a89021a60b3497d24e93ae181fba01aa') + }) + + test('snapshot event stored as session_recording_event', async () => { + await eventsProcessor.processEvent( + 'some-id', + '', + '', + ({ + event: '$snapshot', + properties: { $session_id: 'abcf-efg', $snapshot_data: { timestamp: 123 } }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const events = await getEvents(server) + expect(events.length).toEqual(0) + + const sessionRecordingEvents = await getSessionRecordingEvents(server) + expect(sessionRecordingEvents.length).toBe(1) + + const [event] = sessionRecordingEvents + expect(event.session_id).toEqual('abcf-efg') + expect(event.distinct_id).toEqual('some-id') + expect(event.snapshot_data).toEqual({ timestamp: 123 }) + }) + + test('identify set', async () => { + await createPerson(server, team, ['distinct_id']) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.properties['$set']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(2) + const [person2] = await getPersons(server) + expect(person2.properties).toEqual({ a_prop: 'test-2', b_prop: 'test-2b', c_prop: 'test-1' }) + }) + + test('identify set_once', async () => { + await createPerson(server, team, ['distinct_id']) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-1', c_prop: 'test-1' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + + const [event] = await getEvents(server) + expect(event.properties['$set_once']).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test-1', c_prop: 'test-1' }) + expect(person.is_identified).toEqual(true) + + await eventsProcessor.processEvent( + 'distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + token: team.api_token, + distinct_id: 'distinct_id', + $set_once: { a_prop: 'test-2', b_prop: 'test-2b' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + expect((await getEvents(server)).length).toBe(2) + const [person2] = await getPersons(server) + expect(person2.properties).toEqual({ a_prop: 'test-1', b_prop: 'test-2b', c_prop: 'test-1' }) + }) + + test('distinct with anonymous_id', async () => { + await createPerson(server, team, ['anonymous_id']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + expect((await getEvents(server)).length).toBe(1) + const [event] = await getEvents(server) + expect(event.properties['$set']).toEqual({ a_prop: 'test' }) + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties).toEqual({ a_prop: 'test' }) + + // check no errors as this call can happen multiple times + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + $set: { a_prop: 'test' }, + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + }) + + // This case is likely to happen after signup, for example: + // 1. User browses website with anonymous_id + // 2. User signs up, triggers event with their new_distinct_id (creating a new Person) + // 3. In the frontend, try to alias anonymous_id with new_distinct_id + // Result should be that we end up with one Person with both ID's + test('distinct with anonymous_id which was already created', async () => { + await createPerson(server, team, ['anonymous_id']) + await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person] = await getPersons(server) + expect(await getDistinctIds(server, person)).toEqual(['anonymous_id', 'new_distinct_id']) + expect(person.properties['email']).toEqual('someone@gmail.com') + }) + + test('distinct with multiple anonymous_ids which were already created', async () => { + await createPerson(server, team, ['anonymous_id']) + await createPerson(server, team, ['new_distinct_id'], { email: 'someone@gmail.com' }) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const persons1 = await getPersons(server) + expect(persons1.length).toBe(1) + expect(await getDistinctIds(server, persons1[0])).toEqual(['anonymous_id', 'new_distinct_id']) + expect(persons1[0].properties['email']).toEqual('someone@gmail.com') + + await createPerson(server, team, ['anonymous_id_2']) + + await eventsProcessor.processEvent( + 'new_distinct_id', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: 'anonymous_id_2', + token: team.api_token, + distinct_id: 'new_distinct_id', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const persons2 = await getPersons(server) + expect(persons2.length).toBe(1) + expect(await getDistinctIds(server, persons2[0])).toEqual(['anonymous_id', 'new_distinct_id', 'anonymous_id_2']) + expect(persons2[0].properties['email']).toEqual('someone@gmail.com') + }) + + test('distinct team leakage', async () => { + await createUserTeamAndOrganization( + server.postgres, + 3, + 1002, + '01774e2f-0d01-0000-ee94-9a238640c6ee', + '0174f81e-36f5-0000-7ef8-cc26c1fbab1c' + ) + const team2 = (await getTeams(server))[1] + await createPerson(server, team2, ['2'], { email: 'team2@gmail.com' }) + await createPerson(server, team, ['1', '2']) + + await eventsProcessor.processEvent( + '2', + '', + '', + ({ + event: '$identify', + properties: { + $anon_distinct_id: '1', + token: team.api_token, + distinct_id: '2', + }, + } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const people = await getPersons(server) + expect(people.length).toEqual(2) + expect(people[1].team_id).toEqual(team.id) + expect(people[1].properties).toEqual({}) + expect(await getDistinctIds(server, people[1])).toEqual(['1', '2']) + expect(people[0].team_id).toEqual(team2.id) + expect(await getDistinctIds(server, people[0])).toEqual(['2']) + }) + + test('set is_identified', async () => { + const distinct_id = '777' + const person1 = await createPerson(server, team, [distinct_id]) + expect(person1.is_identified).toBe(false) + + await eventsProcessor.processEvent( + distinct_id, + '', + '', + ({ event: '$identify', properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + const [person2] = await getPersons(server) + expect(person2.is_identified).toBe(true) + }) + + test('team event_properties', async () => { + expect(team.event_properties_numerical).toEqual([]) + + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: 'purchase', properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + + team = await getFirstTeam(server) + expect(team.event_properties).toEqual(['price', 'name', '$ip']) + expect(team.event_properties_numerical).toEqual(['price']) + }) + + test('event name object json', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: { 'event name': 'as object' }, properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.event).toEqual('{"event name":"as object"}') + }) + + test('event name array json', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: ['event name', 'a list'], properties: {} } as any) as PluginEvent, + team.id, + now, + now, + new UUIDT().toString() + ) + const [event] = await getEvents(server) + expect(event.event).toEqual('["event name","a list"]') + }) + + test('long event name substr', async () => { + await eventsProcessor.processEvent( + 'xxx', + '', + '', + ({ event: 'E'.repeat(300), properties: { price: 299.99, name: 'AirPods Pro' } } as any) as PluginEvent, + team.id, + DateTime.utc(), + DateTime.utc(), + 'uuid' + ) + + const [event] = await getEvents(server) + expect(event.event?.length).toBe(200) + }) +} From 576174b9c072bdf34f05a87672439ad3da70e190 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 09:10:49 +0100 Subject: [PATCH 40/47] add query counter --- tests/shared/process-event.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/shared/process-event.test.ts b/tests/shared/process-event.test.ts index f5eca0ca..8e445218 100644 --- a/tests/shared/process-event.test.ts +++ b/tests/shared/process-event.test.ts @@ -44,6 +44,7 @@ export const createProcessEventTests = ( ) => Promise } ) => { + let queryCounter = 0 let team: Team let server: PluginsServer let stopServer: () => Promise @@ -59,6 +60,13 @@ export const createProcessEventTests = ( await server.redis.del(server.PLUGINS_CELERY_QUEUE) await server.redis.del(server.CELERY_DEFAULT_QUEUE) + + const query = server.postgres.query.bind(server.postgres) + server.postgres.query = (queryText: any, values?: any, callback?: any): any => { + queryCounter++ + return query(queryText, values, callback) + } + return [server, stopServer] } @@ -72,6 +80,7 @@ export const createProcessEventTests = ( await resetTestDatabase(testCode) ;[server, stopServer] = await getServer() eventsProcessor = new EventsProcessor(server) + queryCounter = 0 team = await getFirstTeam(server) now = DateTime.utc() }) @@ -107,7 +116,9 @@ export const createProcessEventTests = ( new UUIDT().toString() ) - // TODO: add this back? + expect(queryCounter).toBe(12) + + // TODO: 12 vs 28 is a big difference. Why so? // num_queries = 28 // if settings.EE_AVAILABLE: # extra queries to check for hooks // num_queries += 4 From dfaa2adc73cfec3e3a8062c3573915ee3caaff73 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 09:45:06 +0100 Subject: [PATCH 41/47] clickhouse process event tests v0.1 --- package.json | 3 +- tests/clickhouse/process-event.test.ts | 121 +++++++++++++++++-------- tests/helpers/clickhouse.ts | 8 +- tests/helpers/kafka.ts | 8 +- tests/helpers/sql.ts | 9 +- tests/postgres/process-event.test.ts | 20 ---- tests/shared/process-event.test.ts | 39 +++++--- 7 files changed, 129 insertions(+), 79 deletions(-) diff --git a/package.json b/package.json index d6eee6a1..13304eac 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,8 @@ "prettier:check": "prettier --check .", "prepare": "yarn compile:protobuf", "prepublishOnly": "yarn build", - "setup:dev": "cd ../posthog && dropdb test_posthog && createdb test_posthog && source env/bin/activate && DATABASE_URL=postgres://localhost:5432/test_posthog DEBUG=1 python manage.py migrate" + "setup:dev": "cd ../posthog && (dropdb test_posthog || echo 'no db to drop') && createdb test_posthog && source env/bin/activate && DATABASE_URL=postgres://localhost:5432/test_posthog DEBUG=1 python manage.py migrate", + "setup:dev:ee": "export DEBUG=1 PRIMARY_DB=clickhouse DATABASE_URL=postgres://posthog:posthog@localhost:5439/test_posthog PGPASSWORD=posthog && cd ../posthog && (dropdb -p 5439 -h localhost -U posthog test_posthog || echo 'no db to drop') && createdb -p 5439 -h localhost -U posthog test_posthog && source env/bin/activate && python manage.py migrate && python manage.py migrate_clickhouse" }, "bin": { "posthog-plugin-server": "bin/posthog-plugin-server" diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index 3ee2d62f..d573d4b7 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -1,54 +1,103 @@ -import { PluginsServer } from '../../src/types' +import { + Element, + Event, + Person, + PersonDistinctId, + PluginsServer, + PluginsServerConfig, + PostgresSessionRecordingEvent, +} from '../../src/types' import { createServer } from '../../src/server' import { resetTestDatabase } from '../helpers/sql' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { KafkaCollector, KafkaObserver } from '../helpers/kafka' import { UUIDT } from '../../src/utils' import { DateTime } from 'luxon' +import { createProcessEventTests } from '../shared/process-event.test' jest.setTimeout(180_000) // 3 minute timeout -let server: PluginsServer -let closeServer: () => Promise -const kafkaObserver = new KafkaObserver() +async function getSessionRecordingEvents(server: PluginsServer): Promise { + const result = await server.db.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') + return result.rows as PostgresSessionRecordingEvent[] +} -beforeEach(async () => { - ;[server, closeServer] = await createServer() - await resetTestDatabase(`const processEvent = event => event`) - await resetTestDatabaseClickhouse() -}) -afterEach(() => { - closeServer() -}) +async function getEvents(server: PluginsServer): Promise { + const result = await server.db.postgresQuery('SELECT * FROM posthog_event') + return result.rows as Event[] +} + +async function getPersons(server: PluginsServer): Promise { + const result = await server.db.postgresQuery('SELECT * FROM posthog_person') + return result.rows as Person[] +} + +async function getDistinctIds(server: PluginsServer, person: Person): Promise { + const result = await server.db.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', + [person.id, person.team_id] + ) + return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) +} + +async function getElements(server: PluginsServer, event: Event): Promise { + return (await server.db.postgresQuery('SELECT * FROM posthog_element')).rows +} + +const extraServerConfig: Partial = { + KAFKA_ENABLED: true, + KAFKA_HOSTS: 'kafka:9092', + DATABASE_URL: 'postgres://posthog:posthog@localhost:5439/test_posthog', +} + +describe('process event (clickhouse)', () => { + const kafkaObserver = new KafkaObserver(extraServerConfig) -test('event is passed through', async () => { - const uuid = new UUIDT().toString() - const now = DateTime.utc() - console.log('starting kafka observer') - await kafkaObserver.start() - console.log('sending message') - const kafkaCollector = new KafkaCollector(kafkaObserver) - await kafkaObserver.handOffMessage({ - distinct_id: 'abcd', - ip: '1.1.1.1', - site_url: 'x.com', - team_id: 1, - uuid, - data: { + beforeEach(async () => { + await resetTestDatabaseClickhouse(extraServerConfig) + }) + + const server = createProcessEventTests( + 'clickhouse', + { + getSessionRecordingEvents, + getEvents, + getPersons, + getDistinctIds, + getElements, + }, + extraServerConfig + ) + + test('event is passed through', async () => { + const uuid = new UUIDT().toString() + const now = DateTime.utc() + console.log('starting kafka observer') + await kafkaObserver.start() + console.log('sending message') + const kafkaCollector = new KafkaCollector(kafkaObserver) + await kafkaObserver.handOffMessage({ distinct_id: 'abcd', ip: '1.1.1.1', site_url: 'x.com', team_id: 1, - now: now.toString(), - event: 'test', uuid, - }, - now, - sent_at: null, - }) - console.log('waiting for messages') - const processedMessages = await kafkaCollector.collect(1) + data: { + distinct_id: 'abcd', + ip: '1.1.1.1', + site_url: 'x.com', + team_id: 1, + now: now.toString(), + event: 'test', + uuid, + }, + now, + sent_at: null, + }) + console.log('waiting for messages') + const processedMessages = await kafkaCollector.collect(1) - console.log(processedMessages) - expect(1).toEqual(1) + console.log(processedMessages) + expect(1).toEqual(1) + }) }) diff --git a/tests/helpers/clickhouse.ts b/tests/helpers/clickhouse.ts index aef8f751..9a006dd2 100644 --- a/tests/helpers/clickhouse.ts +++ b/tests/helpers/clickhouse.ts @@ -1,12 +1,14 @@ import { defaultConfig } from '../../src/config' import { ClickHouse } from 'clickhouse' +import { PluginsServerConfig } from '../../src/types' -export async function resetTestDatabaseClickhouse(): Promise { +export async function resetTestDatabaseClickhouse(extraServerConfig: Partial): Promise { + const config = { ...defaultConfig, ...extraServerConfig } const clickhouse = new ClickHouse({ - url: `http://$${defaultConfig.CLICKHOUSE_HOST}`, + url: `http://$${config.CLICKHOUSE_HOST}`, port: 8123, config: { - database: defaultConfig.CLICKHOUSE_DATABASE, + database: config.CLICKHOUSE_DATABASE, }, }) await clickhouse.query('TRUNCATE events').toPromise() diff --git a/tests/helpers/kafka.ts b/tests/helpers/kafka.ts index a7bb0368..15613922 100644 --- a/tests/helpers/kafka.ts +++ b/tests/helpers/kafka.ts @@ -2,8 +2,9 @@ import { EventEmitter } from 'events' import { Kafka, Consumer, logLevel, EachMessagePayload, Producer } from 'kafkajs' import { KAFKA_EVENTS, KAFKA_EVENTS_INGESTION_HANDOFF } from '../../src/ingestion/topics' import { parseRawEventMessage } from '../../src/ingestion/utils' -import { EventMessage } from '../../src/types' +import { EventMessage, PluginsServerConfig } from '../../src/types' import { UUIDT } from '../../src/utils' +import { defaultConfig, overrideWithEnv } from '../../src/config' export class KafkaObserver extends EventEmitter { public kafka: Kafka @@ -12,11 +13,12 @@ export class KafkaObserver extends EventEmitter { private isStarted: boolean - constructor() { + constructor(extraServerConfig: Partial) { super() + const config = { ...overrideWithEnv(defaultConfig, process.env), ...extraServerConfig } this.kafka = new Kafka({ clientId: `plugin-server-test-${new UUIDT()}`, - brokers: (process.env.KAFKA_HOSTS || '').split(','), + brokers: (config.KAFKA_HOSTS || '').split(','), logLevel: logLevel.NOTHING, }) this.producer = this.kafka.producer() diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index 75aac767..92e39a95 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -2,9 +2,14 @@ import { makePluginObjects, commonOrganizationId, commonUserId, commonOrganizati import { defaultConfig } from '../../src/config' import { Pool } from 'pg' import { delay, UUIDT } from '../../src/utils' +import { PluginsServerConfig } from '../../src/types' -export async function resetTestDatabase(code: string): Promise { - const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) +export async function resetTestDatabase( + code: string, + extraServerConfig: Partial = {} +): Promise { + const config = { ...defaultConfig, ...extraServerConfig } + const db = new Pool({ connectionString: config.DATABASE_URL }) const mocks = makePluginObjects(code) await db.query('DELETE FROM posthog_element') await db.query('DELETE FROM posthog_elementgroup') diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index c9c2513f..a96a742a 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -36,36 +36,16 @@ async function getDistinctIds(server: PluginsServer, person: Person): Promise pdi.distinct_id) } -async function getTeams(server: PluginsServer): Promise { - return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows -} - -async function getFirstTeam(server: PluginsServer): Promise { - return (await getTeams(server))[0] -} - async function getElements(server: PluginsServer, event: Event): Promise { return (await server.db.postgresQuery('SELECT * FROM posthog_element')).rows } -async function createPerson( - server: PluginsServer, - team: Team, - distinctIds: string[], - properties: Record = {} -): Promise { - return server.db.createPerson(DateTime.utc(), properties, team.id, null, false, new UUIDT().toString(), distinctIds) -} - describe('process event (postgresql)', () => { createProcessEventTests('postgresql', { getSessionRecordingEvents, getEvents, getPersons, getDistinctIds, - getTeams, - getFirstTeam, getElements, - createPerson, }) }) diff --git a/tests/shared/process-event.test.ts b/tests/shared/process-event.test.ts index 8e445218..2a111f5c 100644 --- a/tests/shared/process-event.test.ts +++ b/tests/shared/process-event.test.ts @@ -9,6 +9,7 @@ import { PersonDistinctId, Element, PostgresSessionRecordingEvent, + PluginsServerConfig, } from '../../src/types' import { createUserTeamAndOrganization, resetTestDatabase } from '../helpers/sql' import { EventsProcessor } from '../../src/ingestion/process-event' @@ -17,6 +18,23 @@ import { UUIDT } from '../../src/utils' jest.setTimeout(600000) // 600 sec timeout +async function getTeams(server: PluginsServer): Promise { + return (await server.db.postgresQuery('SELECT * FROM posthog_team ORDER BY id')).rows +} + +async function getFirstTeam(server: PluginsServer): Promise { + return (await getTeams(server))[0] +} + +async function createPerson( + server: PluginsServer, + team: Team, + distinctIds: string[], + properties: Record = {} +): Promise { + return server.db.createPerson(DateTime.utc(), properties, team.id, null, false, new UUIDT().toString(), distinctIds) +} + export const createProcessEventTests = ( database: 'postgresql' | 'clickhouse', { @@ -24,26 +42,16 @@ export const createProcessEventTests = ( getEvents, getPersons, getDistinctIds, - getTeams, - getFirstTeam, getElements, - createPerson, }: { getSessionRecordingEvents: (server: PluginsServer) => Promise getEvents: (server: PluginsServer) => Promise getPersons: (server: PluginsServer) => Promise getDistinctIds: (server: PluginsServer, person: Person) => Promise - getTeams: (server: PluginsServer) => Promise - getFirstTeam: (server: PluginsServer) => Promise getElements: (server: PluginsServer, event: Event) => Promise - createPerson: ( - server: PluginsServer, - team: Team, - distinctIds: string[], - properties?: Record - ) => Promise - } -) => { + }, + extraServerConfig?: Partial +): PluginsServer => { let queryCounter = 0 let team: Team let server: PluginsServer @@ -56,6 +64,7 @@ export const createProcessEventTests = ( PLUGINS_CELERY_QUEUE: 'test-plugins-celery-queue', CELERY_DEFAULT_QUEUE: 'test-celery-default-queue', LOG_LEVEL: LogLevel.Log, + ...(extraServerConfig ?? {}), }) await server.redis.del(server.PLUGINS_CELERY_QUEUE) @@ -77,7 +86,7 @@ export const createProcessEventTests = ( return event } ` - await resetTestDatabase(testCode) + await resetTestDatabase(testCode, extraServerConfig) ;[server, stopServer] = await getServer() eventsProcessor = new EventsProcessor(server) queryCounter = 0 @@ -1032,4 +1041,6 @@ export const createProcessEventTests = ( const [event] = await getEvents(server) expect(event.event?.length).toBe(200) }) + + return server! } From 9049790d7b3842be1b759eb3e5b2aeee357ffc43 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 09:53:05 +0100 Subject: [PATCH 42/47] fix vm test --- tests/postgres/vm.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts index 77245030..dfa6ff41 100644 --- a/tests/postgres/vm.test.ts +++ b/tests/postgres/vm.test.ts @@ -716,7 +716,7 @@ test('posthog in runEvery', async () => { expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue - const mockClientInstance = (Client as any).mock.instances[0] + const mockClientInstance = (Client as any).mock.instances[1] const mockSendTask = mockClientInstance.sendTask expect(mockSendTask.mock.calls[0][0]).toEqual('posthog.tasks.process_event.process_event_with_plugins') @@ -755,7 +755,7 @@ test('posthog in runEvery with timestamp', async () => { expect((Client as any).mock.calls[0][1]).toEqual(mockServer.CELERY_DEFAULT_QUEUE) // webhook to celery queue expect((Client as any).mock.calls[1][1]).toEqual(mockServer.PLUGINS_CELERY_QUEUE) // events out to start of plugin queue - const mockClientInstance = (Client as any).mock.instances[0] + const mockClientInstance = (Client as any).mock.instances[1] const mockSendTask = mockClientInstance.sendTask expect(mockSendTask.mock.calls[0][0]).toEqual('posthog.tasks.process_event.process_event_with_plugins') From 13ca116053a6f60bc217b582abe4f6fd88852529 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 10:02:18 +0100 Subject: [PATCH 43/47] fix "Your test suite must contain at least one test." error for shared tests --- tests/clickhouse/process-event.test.ts | 4 +--- tests/postgres/process-event.test.ts | 14 ++------------ .../{process-event.test.ts => process-event.ts} | 1 - 3 files changed, 3 insertions(+), 16 deletions(-) rename tests/shared/{process-event.test.ts => process-event.ts} (99%) diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index d573d4b7..f2f85bbb 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -7,13 +7,11 @@ import { PluginsServerConfig, PostgresSessionRecordingEvent, } from '../../src/types' -import { createServer } from '../../src/server' -import { resetTestDatabase } from '../helpers/sql' import { resetTestDatabaseClickhouse } from '../helpers/clickhouse' import { KafkaCollector, KafkaObserver } from '../helpers/kafka' import { UUIDT } from '../../src/utils' import { DateTime } from 'luxon' -import { createProcessEventTests } from '../shared/process-event.test' +import { createProcessEventTests } from '../shared/process-event' jest.setTimeout(180_000) // 3 minute timeout diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index a96a742a..07fe850e 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -1,15 +1,5 @@ -import { - PluginsServer, - Team, - Event, - Person, - PersonDistinctId, - Element, - PostgresSessionRecordingEvent, -} from '../../src/types' -import { DateTime } from 'luxon' -import { UUIDT } from '../../src/utils' -import { createProcessEventTests } from '../shared/process-event.test' +import { PluginsServer, Event, Person, PersonDistinctId, Element, PostgresSessionRecordingEvent } from '../../src/types' +import { createProcessEventTests } from '../shared/process-event' jest.setTimeout(600000) // 600 sec timeout diff --git a/tests/shared/process-event.test.ts b/tests/shared/process-event.ts similarity index 99% rename from tests/shared/process-event.test.ts rename to tests/shared/process-event.ts index 2a111f5c..1417dc52 100644 --- a/tests/shared/process-event.test.ts +++ b/tests/shared/process-event.ts @@ -6,7 +6,6 @@ import { Team, Event, Person, - PersonDistinctId, Element, PostgresSessionRecordingEvent, PluginsServerConfig, From 1bf4008ee736f1d573949562c50ba8321d956cf4 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 1 Feb 2021 10:08:26 +0100 Subject: [PATCH 44/47] also run non-ingestion tests under test:postgres --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 13304eac..bed5959b 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "main": "dist/src/index.js", "scripts": { "test": "jest --runInBand tests/**/*.test.ts", - "test:postgres": "yarn test --testPathIgnorePatterns '.*/clickhouse'", - "test:clickhouse": "yarn test --testPathIgnorePatterns '.*/postgres'", + "test:postgres": "jest --runInBand tests/postgres/*.test.ts tests/*.test.ts", + "test:clickhouse": "jest --runInBand tests/clickhouse/*.test.ts", "benchmark": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/", "start": "yarn start:dev", "start:dist": "node dist/src/index.js --base-dir ../posthog", From 6aad71430537d48f805c28cd09c42bd39994f3e7 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 1 Feb 2021 10:28:58 +0100 Subject: [PATCH 45/47] Clean up utils.ts --- src/ingestion/utils.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ingestion/utils.ts b/src/ingestion/utils.ts index 409b6b0d..fdab780b 100644 --- a/src/ingestion/utils.ts +++ b/src/ingestion/utils.ts @@ -69,24 +69,24 @@ export function sanitizeEventName(eventName: any): string { return eventName.substr(0, 200) } -// escape utf-8 characters into `\u1234` -function jsonEscapeUTF8(s: string): string { +/** Escape UTF-8 characters into `\u1234`. */ +function jsonEscapeUtf8(s: string): string { return s.replace(/[^\x20-\x7F]/g, (x) => '\\u' + ('000' + x.codePointAt(0)?.toString(16)).slice(-4)) } -// produce output compatible to that of python's json.dumps -function pythonDumps(obj: any): string { +/** Produce output compatible with that of Python's `json.dumps`. */ +function jsonDumps(obj: any): string { if (typeof obj === 'object' && obj !== null) { if (Array.isArray(obj)) { - return `[${obj.map(pythonDumps).join(', ')}]` // space after comma + return `[${obj.map(jsonDumps).join(', ')}]` // space after comma } else { return `{${Object.keys(obj) // no space after '{' or before '}' .sort() // must sort the keys of the object! - .map((k) => `${pythonDumps(k)}: ${pythonDumps(obj[k])}`) // space after ':' + .map((k) => `${jsonDumps(k)}: ${jsonDumps(obj[k])}`) // space after ':' .join(', ')}}` // space after ',' } } else if (typeof obj === 'string') { - return jsonEscapeUTF8(JSON.stringify(obj)) + return jsonEscapeUtf8(JSON.stringify(obj)) } else { return JSON.stringify(obj) } @@ -105,7 +105,7 @@ export function hashElements(elements: Element[]): string { order: element.order ?? null, })) - const serializedString = pythonDumps(elementsList) + const serializedString = jsonDumps(elementsList) return crypto.createHash('md5').update(serializedString).digest('hex') } From a8c76aff78ddc8c95bd5d899d408394e549a70c8 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 1 Feb 2021 10:36:12 +0100 Subject: [PATCH 46/47] Clean up TimestampFormat --- src/ingestion/process-event.ts | 2 +- src/types.ts | 2 +- src/utils.ts | 11 +++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ingestion/process-event.ts b/src/ingestion/process-event.ts index 1f0aa443..73a8e0db 100644 --- a/src/ingestion/process-event.ts +++ b/src/ingestion/process-event.ts @@ -434,7 +434,7 @@ export class EventsProcessor { ): Promise { const timestampString = castTimestampOrNow( timestamp, - this.kafkaProducer ? TimestampFormat.Clickhouse : TimestampFormat.ISO + this.kafkaProducer ? TimestampFormat.ClickHouse : TimestampFormat.ISO ) const elementsChain = elements && elements.length ? elementsToString(elements) : '' diff --git a/src/types.ts b/src/types.ts index 5fbf55f1..e457f674 100644 --- a/src/types.ts +++ b/src/types.ts @@ -311,6 +311,6 @@ export interface PostgresSessionRecordingEvent extends Omit { From 6a353b689e8e846765f8928ad2a8c79ad2c45f77 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 1 Feb 2021 10:52:28 +0100 Subject: [PATCH 47/47] Put get* type Postgres functions in DB class --- src/db.ts | 49 +++++++++++++++++++++++++- tests/clickhouse/process-event.test.ts | 37 +++---------------- tests/postgres/process-event.test.ts | 37 +++---------------- 3 files changed, 58 insertions(+), 65 deletions(-) diff --git a/src/db.ts b/src/db.ts index 95256404..7b4f430c 100644 --- a/src/db.ts +++ b/src/db.ts @@ -5,7 +5,15 @@ import { DateTime } from 'luxon' import { Pool, QueryConfig, QueryResult, QueryResultRow } from 'pg' import { KAFKA_PERSON, KAFKA_PERSON_UNIQUE_ID } from './ingestion/topics' import { unparsePersonPartial } from './ingestion/utils' -import { Person, PersonDistinctId, RawPerson, RawOrganization, Team } from './types' +import { + Person, + PersonDistinctId, + RawPerson, + RawOrganization, + Team, + PostgresSessionRecordingEvent, + Event, +} from './types' import { castTimestampOrNow, sanitizeSqlIdentifier, UUIDT } from './utils' /** The recommended way of accessing the database. */ @@ -30,6 +38,13 @@ export class DB { return this.postgres.query(queryTextOrConfig, values) } + // Person + + public async fetchPersons(): Promise { + const result = await this.postgresQuery('SELECT * FROM posthog_person') + return result.rows as Person[] + } + public async fetchPerson(teamId: number, distinctId: string): Promise { const selectResult = await this.postgresQuery( `SELECT @@ -122,6 +137,16 @@ export class DB { } } + // PersonDistinctId + + public async fetchDistinctIdValues(person: Person): Promise { + const result = await this.postgresQuery( + 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', + [person.id, person.team_id] + ) + return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) + } + public async addDistinctId(person: Person, distinctId: string): Promise { const insertResult = await this.postgresQuery( 'INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id) VALUES ($1, $2, $3) RETURNING *', @@ -155,6 +180,8 @@ export class DB { } } + // Organization + public async fetchOrganization(organizationId: string): Promise { const selectResult = await this.postgresQuery(`SELECT * FROM posthog_organization WHERE id $1`, [ organizationId, @@ -162,4 +189,24 @@ export class DB { const rawOrganization: RawOrganization = selectResult.rows[0] return rawOrganization } + + // Event + + public async fetchEvents(): Promise { + const result = await this.postgresQuery('SELECT * FROM posthog_event') + return result.rows as Event[] + } + + // SessionRecordingEvent + + public async fetchSessionRecordingEvents(): Promise { + const result = await this.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') + return result.rows as PostgresSessionRecordingEvent[] + } + + // Element + + public async fetchElements(): Promise { + return (await this.postgresQuery('SELECT * FROM posthog_element')).rows + } } diff --git a/tests/clickhouse/process-event.test.ts b/tests/clickhouse/process-event.test.ts index f2f85bbb..b86ed4be 100644 --- a/tests/clickhouse/process-event.test.ts +++ b/tests/clickhouse/process-event.test.ts @@ -15,33 +15,6 @@ import { createProcessEventTests } from '../shared/process-event' jest.setTimeout(180_000) // 3 minute timeout -async function getSessionRecordingEvents(server: PluginsServer): Promise { - const result = await server.db.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') - return result.rows as PostgresSessionRecordingEvent[] -} - -async function getEvents(server: PluginsServer): Promise { - const result = await server.db.postgresQuery('SELECT * FROM posthog_event') - return result.rows as Event[] -} - -async function getPersons(server: PluginsServer): Promise { - const result = await server.db.postgresQuery('SELECT * FROM posthog_person') - return result.rows as Person[] -} - -async function getDistinctIds(server: PluginsServer, person: Person): Promise { - const result = await server.db.postgresQuery( - 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', - [person.id, person.team_id] - ) - return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) -} - -async function getElements(server: PluginsServer, event: Event): Promise { - return (await server.db.postgresQuery('SELECT * FROM posthog_element')).rows -} - const extraServerConfig: Partial = { KAFKA_ENABLED: true, KAFKA_HOSTS: 'kafka:9092', @@ -58,11 +31,11 @@ describe('process event (clickhouse)', () => { const server = createProcessEventTests( 'clickhouse', { - getSessionRecordingEvents, - getEvents, - getPersons, - getDistinctIds, - getElements, + getSessionRecordingEvents: (server) => server.db.fetchSessionRecordingEvents(), + getEvents: (server) => server.db.fetchEvents(), + getPersons: (server) => server.db.fetchPersons(), + getDistinctIds: (server, person) => server.db.fetchDistinctIdValues(person), + getElements: (server) => server.db.fetchElements(), }, extraServerConfig ) diff --git a/tests/postgres/process-event.test.ts b/tests/postgres/process-event.test.ts index 07fe850e..d404d826 100644 --- a/tests/postgres/process-event.test.ts +++ b/tests/postgres/process-event.test.ts @@ -3,39 +3,12 @@ import { createProcessEventTests } from '../shared/process-event' jest.setTimeout(600000) // 600 sec timeout -async function getSessionRecordingEvents(server: PluginsServer): Promise { - const result = await server.db.postgresQuery('SELECT * FROM posthog_sessionrecordingevent') - return result.rows as PostgresSessionRecordingEvent[] -} - -async function getEvents(server: PluginsServer): Promise { - const result = await server.db.postgresQuery('SELECT * FROM posthog_event') - return result.rows as Event[] -} - -async function getPersons(server: PluginsServer): Promise { - const result = await server.db.postgresQuery('SELECT * FROM posthog_person') - return result.rows as Person[] -} - -async function getDistinctIds(server: PluginsServer, person: Person): Promise { - const result = await server.db.postgresQuery( - 'SELECT * FROM posthog_persondistinctid WHERE person_id=$1 and team_id=$2 ORDER BY id', - [person.id, person.team_id] - ) - return (result.rows as PersonDistinctId[]).map((pdi) => pdi.distinct_id) -} - -async function getElements(server: PluginsServer, event: Event): Promise { - return (await server.db.postgresQuery('SELECT * FROM posthog_element')).rows -} - describe('process event (postgresql)', () => { createProcessEventTests('postgresql', { - getSessionRecordingEvents, - getEvents, - getPersons, - getDistinctIds, - getElements, + getSessionRecordingEvents: (server) => server.db.fetchSessionRecordingEvents(), + getEvents: (server) => server.db.fetchEvents(), + getPersons: (server) => server.db.fetchPersons(), + getDistinctIds: (server, person) => server.db.fetchDistinctIdValues(person), + getElements: (server) => server.db.fetchElements(), }) })