From 28c1fd2a5d00c071967f61ebd90384a8573f6346 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Mon, 30 Dec 2024 18:05:20 +0000 Subject: [PATCH 1/4] Migrate more core spec tests --- core/main_test.ts | 261 +++++++++++++++++++--- core/session.ts | 4 +- tests/core/core.spec.ts | 474 ---------------------------------------- 3 files changed, 238 insertions(+), 501 deletions(-) diff --git a/core/main_test.ts b/core/main_test.ts index 1c8d6ebf7..8b9d1d772 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -89,7 +89,7 @@ suite("@dataform/core", ({ afterEach }) => { }); }); - test("resolve fails", () => { + test("fails when cannot resolve", () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), @@ -106,6 +106,30 @@ suite("@dataform/core", ({ afterEach }) => { ).deep.equals(`Could not resolve "e"`); }); + test("fails when ambiguous resolve", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.js"), + ` +publish("a", {"schema": "foo"}) +publish("a", {"schema": "bar"}) +publish("b", {"schema": "foo"}).dependencies("a")` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors?.map(error => error.message) + ).deep.equals([ + `Ambiguous Action name: {\"name\":\"a\",\"includeDependentAssertions\":false}. Did you mean one of: foo.a, bar.a.` + ]); + }); + suite("context methods", () => { [ TestConfigs.bigqueryWithDefaultProjectAndDataset, @@ -193,10 +217,8 @@ actions: `Action target names cannot include '.'` ]); }); - }); - suite("actions", () => { - test("disabled", () => { + test("fails when non-unique target", () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), @@ -204,24 +226,184 @@ actions: ); fs.mkdirSync(path.join(projectDir, "definitions")); fs.writeFileSync( - path.join(projectDir, "definitions/table.sqlx"), - "config { type: 'table', disabled: true }" + path.join(projectDir, "definitions/file.js"), + ` +publish("table") +publish("table")` ); + + const result = runMainInVm( + coreExecutionRequestFromPath( + projectDir, + dataform.ProjectConfig.create({ + defaultSchema: "otherDataset" + }) + ) + ); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors?.map(error => error.message) + ).deep.equals([ + `Duplicate action name detected. Names within a schema must be unique across tables, declarations, assertions, and operations:\n\"{\"schema\":\"otherDataset\",\"name\":\"table\",\"database\":\"defaultProject\"}\"`, + `Duplicate canonical target detected. Canonical targets must be unique across tables, declarations, assertions, and operations:\n\"{\"schema\":\"otherDataset\",\"name\":\"table\",\"database\":\"defaultProject\"}\"`, + `Duplicate action name detected. Names within a schema must be unique across tables, declarations, assertions, and operations:\n\"{\"schema\":\"otherDataset\",\"name\":\"table\",\"database\":\"defaultProject\"}\"`, + `Duplicate canonical target detected. Canonical targets must be unique across tables, declarations, assertions, and operations:\n\"{\"schema\":\"otherDataset\",\"name\":\"table\",\"database\":\"defaultProject\"}\"` + ]); + }); + + test("fails when circular dependencies", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( - path.join(projectDir, "definitions/operation.sqlx"), - "config { type: 'operations', disabled: false }" + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.js"), + ` +publish("a").dependencies("b") +publish("b").dependencies("a")` ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors?.map(error => error.message) + ).deep.equals([ + `Circular dependency detected in chain: [{\"database\":\"defaultProject\",\"name\":\"a\",\"schema\":\"defaultDataset\"} > {\"database\":\"defaultProject\",\"name\":\"b\",\"schema\":\"defaultDataset\"} > defaultProject.defaultDataset.a]` + ]); + }); + + test("fails when missing dependency", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( - path.join(projectDir, "definitions/assertion.sqlx"), - "config { type: 'assertion', disabled: true }" + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/file.sql"), "unused"); + fs.writeFileSync( + path.join(projectDir, "definitions/file.js"), + ` +publish("a").dependencies("b")` ); const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); - expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.tables[0].disabled)).equals(true); - expect(asPlainObject(result.compile.compiledGraph.operations[0].disabled)).equals(false); - expect(asPlainObject(result.compile.compiledGraph.assertions[0].disabled)).equals(true); + expect( + result.compile.compiledGraph.graphErrors.compilationErrors?.map(error => error.message) + ).deep.equals([ + `Missing dependency detected: Action \"defaultProject.defaultDataset.a\" depends on \"{\"name\":\"b\",\"includeDependentAssertions\":false}\" which does not exist` + ]); + }); + + test("semi-colons at the end of SQL statements throws", () => { + // If this didn't happen, then the generated SQL could be incorrect + // because of being broken up by semi-colons. + + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.js"), + ` +publish("a", "SELECT 1;\\n"); +publish("b", "SELECT 1;");` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors?.map(error => error.message) + ).deep.equals([ + "Semi-colons are not allowed at the end of SQL statements.", + "Semi-colons are not allowed at the end of SQL statements." + ]); + }); + }); + + suite("actions", () => { + const getActionsFromResult = (tableType: string, result: dataform.CoreExecutionResponse) => { + switch (tableType) { + case "table": + case "view": + case "incremental": + return result.compile.compiledGraph.tables; + case "operations": + return result.compile.compiledGraph.operations; + case "assertion": + return result.compile.compiledGraph.assertions; + default: + throw Error(`Unexpected table type: ${tableType}`); + } + }; + + ["table", "view", "incremental", "operations", "assertion"].forEach(tableType => { + test(`${tableType} disabled`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, `definitions/${tableType}.sqlx`), + `config { type: '${tableType}', disabled: true }` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(getActionsFromResult(tableType, result)[0]?.disabled)).is.true; + }); + + test(`${tableType} target can be overridden by project config override`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.sqlx"), + // If change, then change test "action configs assertions can be loaded". + ` +config { + type: "${tableType}", + name: "name", +} +SELECT 1` + ); + + const result = runMainInVm( + coreExecutionRequestFromPath( + projectDir, + dataform.ProjectConfig.create({ + defaultDatabase: "otherProject", + defaultSchema: "otherDataset", + assertionSchema: "otherDataset", + tablePrefix: "prefix" + }) + ) + ); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(getActionsFromResult(tableType, result)[0]?.target)).deep.equals({ + database: "otherProject", + schema: "otherDataset", + name: "prefix_name" + }); + expect( + asPlainObject(getActionsFromResult(tableType, result)[0]?.canonicalTarget) + ).deep.equals({ + database: "otherProject", + schema: "otherDataset", + name: "name" + }); + }); }); }); @@ -356,7 +538,7 @@ quotes }); suite("workflow settings", () => { - test(`main succeeds when a valid workflow_settings.yaml is present`, () => { + test(`valid workflow_settings.yaml is present`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), @@ -377,7 +559,7 @@ quotes }); // dataform.json for workflow settings is deprecated, but still currently supported. - test(`main succeeds when a valid dataform.json is present`, () => { + test(`a valid dataform.json is present`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync(path.join(projectDir, "dataform.json"), VALID_DATAFORM_JSON); @@ -393,7 +575,7 @@ quotes ); }); - test(`main fails when no workflow settings file is present`, () => { + test(`fails when no workflow settings file is present`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); expect(() => runMainInVm(coreExecutionRequestFromPath(projectDir))).to.throw( @@ -401,7 +583,7 @@ quotes ); }); - test(`main fails when both workflow settings and dataform.json files are present`, () => { + test(`fails when both workflow settings and dataform.json files are present`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync(path.join(projectDir, "dataform.json"), VALID_DATAFORM_JSON); fs.writeFileSync( @@ -414,7 +596,7 @@ quotes ); }); - test(`main fails when workflow_settings.yaml cannot be represented in JSON format`, () => { + test(`fails when workflow_settings.yaml cannot be represented in JSON format`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync(path.join(projectDir, "workflow_settings.yaml"), "&*19132sdS:asd:"); @@ -423,7 +605,7 @@ quotes ); }); - test(`main fails when workflow settings fails to be parsed`, () => { + test(`fails when workflow settings fails to be parsed`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), @@ -437,7 +619,7 @@ someKey: and an extra: colon ); }); - test(`main fails when dataform.json is an invalid json file`, () => { + test(`fails when dataform.json is an invalid json file`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync(path.join(projectDir, "dataform.json"), '{keyWithNoQuotes: "validValue"}'); @@ -446,7 +628,7 @@ someKey: and an extra: colon ); }); - test(`main fails when a valid workflow_settings.yaml contains unknown fields`, () => { + test(`fails when a valid workflow_settings.yaml contains unknown fields`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), @@ -458,7 +640,7 @@ someKey: and an extra: colon ); }); - test(`main fails when a valid workflow_settings.yaml base level is an array`, () => { + test(`fails when a valid workflow_settings.yaml base level is an array`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync(path.join(projectDir, "workflow_settings.yaml"), "- someArrayEntry"); @@ -467,7 +649,7 @@ someKey: and an extra: colon ); }); - test(`main fails when a valid dataform.json contains unknown fields`, () => { + test(`fails when a valid dataform.json contains unknown fields`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "dataform.json"), @@ -479,6 +661,24 @@ someKey: and an extra: colon ); }); + test("fails when defaultLocation is not present in workflow_settings.yaml", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors?.map(error => error.message) + ).deep.equals([ + `A defaultLocation is required for BigQuery. This can be configured in workflow_settings.yaml.` + ]); + }); + test(`workflow settings and project config overrides are merged and applied within SQLX files`, () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( @@ -3512,9 +3712,18 @@ publish("name", { }); }); -function coreExecutionRequestFromPath(projectDir: string): dataform.CoreExecutionRequest { +function coreExecutionRequestFromPath( + projectDir: string, + projectConfigOverride?: dataform.ProjectConfig +): dataform.CoreExecutionRequest { return dataform.CoreExecutionRequest.create({ - compile: { compileConfig: { projectDir, filePaths: walkDirectoryForFilenames(projectDir) } } + compile: { + compileConfig: { + projectDir, + filePaths: walkDirectoryForFilenames(projectDir), + projectConfigOverride + } + } }); } diff --git a/core/session.ts b/core/session.ts index cea1746b6..c315f37f1 100644 --- a/core/session.ts +++ b/core/session.ts @@ -717,7 +717,9 @@ export class Session { if (isNonUniqueTarget) { this.compileError( new Error( - `Duplicate action name detected. Names within a schema must be unique across tables, declarations, assertions, and operations` + `Duplicate action name detected. Names within a schema must be unique across tables, declarations, assertions, and operations:\n"${JSON.stringify( + action.target + )}"` ), action.fileName, action.target diff --git a/tests/core/core.spec.ts b/tests/core/core.spec.ts index 441951291..fd3afdb34 100644 --- a/tests/core/core.spec.ts +++ b/tests/core/core.spec.ts @@ -37,155 +37,6 @@ class TestConfigs { } suite("@dataform/core", () => { - test("canonical targets", () => { - const originalConfig = dataform.ProjectConfig.create({ - warehouse: "bigquery", - defaultSchema: "schema", - defaultDatabase: "database", - schemaSuffix: "dev", - tablePrefix: "dev" - }); - const overrideConfig = dataform.ProjectConfig.create({ - ...originalConfig, - defaultSchema: "otherschema", - defaultDatabase: "otherdatabase" - }); - const session = new Session(path.dirname(__filename), overrideConfig, originalConfig); - session.publish("dataset"); - session.assert("assertion"); - session.declare({ name: "declaration" }); - session.operate("operation"); - - const graph = session.compile(); - expect( - [ - ...graph.tables, - ...graph.assertions, - ...graph.declarations, - ...graph.operations - ].map(action => dataform.Target.create(action.canonicalTarget).toJSON()) - ).deep.equals([ - { - database: "database", - name: "dataset", - schema: "schema" - }, - { - database: "database", - name: "assertion", - schema: "schema" - }, - { - database: "database", - name: "declaration", - schema: "schema" - }, - { - database: "database", - name: "operation", - schema: "schema" - } - ]); - }); - - test("non-unique canonical targets fails", () => { - const originalConfig = dataform.ProjectConfig.create({ - warehouse: "bigquery", - defaultSchema: "schema", - defaultDatabase: "database", - defaultLocation: "US" - }); - const overrideConfig = dataform.ProjectConfig.create({ - ...originalConfig, - defaultSchema: "otherschema" - }); - const session = new Session(path.dirname(__filename), overrideConfig, originalConfig); - session - .publish("view", { - type: "view" - }) - .query("query"); - session.publish("view", { - type: "view", - schema: "schema" - }); - const graph = session.compile(); - expect(graph.graphErrors.compilationErrors.map(error => error.message)).deep.equals( - Array(2).fill( - 'Duplicate canonical target detected. Canonical targets must be unique across tables, declarations, assertions, and operations:\n"{"schema":"schema","name":"view","database":"database"}"' - ) - ); - }); - - test("validation_type_incremental", () => { - const sessionSuccess = new Session(path.dirname(__filename), TestConfigs.bigquery); - sessionSuccess - .publish("exampleSuccess1", { - type: "incremental" - }) - .where("test1"); - sessionSuccess - .publish("exampleSuccess2", ctx => `select field as 1`) - .where("test2") - .type("incremental"); - const cgSuccess = sessionSuccess.compile(); - expect(cgSuccess.graphErrors.compilationErrors).deep.equals([]); - }); - - test("validation_type", () => { - const sessionSuccess = new Session(path.dirname(__filename), TestConfigs.bigquery); - sessionSuccess.publish("exampleSuccess1", { type: "table" }); - sessionSuccess.publish("exampleSuccess2", { type: "view" }); - sessionSuccess.publish("exampleSuccess3", { type: "incremental" }).where("test"); - const cgSuccess = sessionSuccess.compile(); - expect(cgSuccess.graphErrors.compilationErrors).deep.equals([]); - - const sessionFail = new Session(path.dirname(__filename), TestConfigs.bigquery); - sessionFail.publish("exampleFail", JSON.parse('{"type": "ta ble"}')); - const cgFail = sessionFail.compile(); - - expect(cgFail.toJSON().graphErrors.compilationErrors).deep.equals([ - { - fileName: "core.spec.js", - actionName: "schema.exampleFail", - actionTarget: { schema: "schema", name: "exampleFail" }, - message: - 'Wrong type of table detected. Should only use predefined types: "table" | "view" | "incremental"' - } - ]); - - const err = cgFail.graphErrors.compilationErrors.find( - e => e.actionName === "schema.exampleFail" - ); - expect(err) - .to.have.property("message") - .that.matches(/Wrong type of table/); - }); - - test("validation_bigquery_fail", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("example_materialized_view", { - type: "table", - materialized: true - }); - - const expectedResults = [ - { - name: "schema.example_materialized_view", - message: "The 'materialized' option is only valid for BigQuery views" - } - ]; - - const graph = session.compile(); - - expect( - graph.graphErrors.compilationErrors.map(compilationError => ({ - name: compilationError.actionName, - message: compilationError.message - })) - ).to.have.deep.members(expectedResults); - }); - test("validation_bigquery_fail", () => { const session = new Session(path.dirname(__filename), TestConfigs.bigquery); session.publish("example_partitionBy_view_fail", { @@ -358,156 +209,6 @@ suite("@dataform/core", () => { expect(graph.graphErrors.compilationErrors).to.deep.equals([]); }); - [ - TestConfigs.bigquery, - TestConfigs.bigqueryWithSchemaSuffix, - TestConfigs.bigqueryWithTablePrefix - ].forEach(testConfig => { - test(`ref with prefix "${testConfig.tablePrefix}" and suffix "${testConfig.schemaSuffix}"`, () => { - const session = new Session(path.dirname(__filename), testConfig); - const suffix = testConfig.schemaSuffix ? `_${testConfig.schemaSuffix}` : ""; - const prefix = testConfig.tablePrefix ? `${testConfig.tablePrefix}_` : ""; - - session.publish(`a`, _ => "select 1 as test"); - session.publish(`b`, ctx => `select * from ${ctx.ref("a")}`); - session.publish(`c`, ctx => `select * from ${ctx.ref(undefined)}`); - session.publish(`d`, ctx => `select * from ${ctx.ref({ schema: "schema", name: "a" })}`); - session.publish(`g`, ctx => `select * from ${ctx.ref("schema", "a")}`); - session.publish(`h`, ctx => `select * from ${ctx.ref(["schema", "a"])}`); - session - .publish("e", { - schema: "foo" - }) - .query(_ => "select 1 as test"); - session.publish("f", ctx => `select * from ${ctx.ref("e")}`); - - const graph = session.compile(); - - const tableNames = graph.tables.map(table => targetAsReadableString(table.target)); - - const baseEqlArray = [ - "schema.a", - "schema.b", - "schema.c", - "schema.d", - "schema.g", - "schema.h", - "foo.e", - "schema.f" - ]; - - expect(tableNames).eql( - baseEqlArray.map(item => { - if (testConfig.tablePrefix) { - const separatedItems = item.split("."); - separatedItems[1] = `${prefix}${separatedItems[1]}`; - return separatedItems.join("."); - } - - if (testConfig.schemaSuffix) { - const separatedItems = item.split("."); - separatedItems[0] = `${separatedItems[0]}${suffix}`; - return separatedItems.join("."); - } - - return item; - }) - ); - - expect( - graph.tables - .find(table => targetAsReadableString(table.target) === `schema${suffix}.${prefix}b`) - .dependencyTargets.map(dependency => targetAsReadableString(dependency)) - ).eql([`schema${suffix}.${prefix}a`]); - expect( - graph.tables - .find(table => targetAsReadableString(table.target) === `schema${suffix}.${prefix}d`) - .dependencyTargets.map(dependency => targetAsReadableString(dependency)) - ).eql([`schema${suffix}.${prefix}a`]); - expect( - graph.tables - .find(table => targetAsReadableString(table.target) === `schema${suffix}.${prefix}g`) - .dependencyTargets.map(dependency => targetAsReadableString(dependency)) - ).eql([`schema${suffix}.${prefix}a`]); - expect( - graph.tables - .find(table => targetAsReadableString(table.target) === `schema${suffix}.${prefix}h`) - .dependencyTargets.map(dependency => targetAsReadableString(dependency)) - ).eql([`schema${suffix}.${prefix}a`]); - expect( - graph.tables - .find(table => targetAsReadableString(table.target) === `schema${suffix}.${prefix}f`) - .dependencyTargets.map(dependency => targetAsReadableString(dependency)) - ).eql([`foo${suffix}.${prefix}e`]); - - const errors = graph.graphErrors.compilationErrors.map(item => item.message); - expect(errors).includes("Action name is not specified"); - expect(graph.graphErrors.compilationErrors.length).eql(1); - }); - }); - - [ - { testConfig: TestConfigs.bigquery, target: "schema" }, - { testConfig: TestConfigs.bigqueryWithSchemaSuffix, target: "schema_suffix" } - ].forEach(({ testConfig, target }) => { - test(`schema/suffix: "${target}"`, () => { - const session = new Session(path.dirname(__filename), testConfig); - session.publish("test", { type: "table" }).query(ctx => ctx.schema()); - - const graph = session.compile(); - - const testTable = graph.tables.find( - table => targetAsReadableString(table.target) === `${target}.test` - ); - - expect(testTable.query).deep.equals(target); - }); - }); - - [ - { testConfig: TestConfigs.bigquery, target: "schema.test", name: "test" }, - { - testConfig: TestConfigs.bigqueryWithTablePrefix, - target: "schema.prefix_test", - name: "prefix_test" - } - ].forEach(({ testConfig, target, name }) => { - test(`name/prefix: "${target}"`, () => { - const session = new Session(path.dirname(__filename), testConfig); - session.publish("test", { type: "table" }).query(ctx => ctx.name()); - - const graph = session.compile(); - - const testTable = graph.tables.find(table => targetAsReadableString(table.target) === target); - - expect(testTable.query).deep.equals(name); - }); - }); - - [ - { - testConfig: TestConfigs.bigqueryWithDefaultDatabase, - target: "default-database.schema.test", - database: "default-database" - }, - { - testConfig: TestConfigs.bigqueryWithDefaultDatabaseAndSuffix, - target: "default-database_suffix.schema.test", - database: "default-database_suffix" - } - ].forEach(({ testConfig, target, database }) => { - test(`database/suffix: "${target}"`, () => { - const session = new Session(path.dirname(__filename), testConfig); - session.publish("test", { type: "table" }).query(ctx => ctx.database()); - - const graph = session.compile(); - - const testTable = graph.tables.find(table => targetAsReadableString(table.target) === target); - - expect(testTable.query).deep.equals(database); - }); - }); - test(`database fails when undefined`, () => { const session = new Session(path.dirname(__filename), TestConfigs.bigquery); session.publish("test", { type: "table" }).query(ctx => ctx.database()); @@ -552,23 +253,6 @@ suite("operate", () => { expect(graph.operations[1].queries).deep.equals(["select * from `schema.operate-1`"]); }); - [ - { testConfig: TestConfigs.bigquery, finalizedSchema: "schema" }, - { - testConfig: TestConfigs.bigqueryWithSchemaSuffix, - finalizedSchema: "schema_suffix" - } - ].forEach(({ testConfig, finalizedSchema }) => { - test(`schema with suffix: "${finalizedSchema}"`, () => { - const session = new Session(path.dirname(__filename), testConfig); - session.operate("operate-1", ctx => ctx.schema()).hasOutput(true); - - const graph = session.compile(); - - expect(graph.operations[0].queries).deep.equals([finalizedSchema]); - }); - }); - [ { testConfig: TestConfigs.bigquery, finalizedName: "operate-1" }, { @@ -585,162 +269,4 @@ suite("operate", () => { expect(graph.operations[0].queries).deep.equals([finalizedName]); }); }); - - [ - { - testConfig: TestConfigs.bigqueryWithDefaultDatabase, - finalizedDatabase: "default-database" - }, - { - testConfig: TestConfigs.bigqueryWithDefaultDatabaseAndSuffix, - finalizedDatabase: "default-database_suffix" - } - ].forEach(({ testConfig, finalizedDatabase }) => { - test(`database with suffix: "${finalizedDatabase}"`, () => { - const session = new Session(path.dirname(__filename), testConfig); - session.operate("operate-1", ctx => ctx.database()).hasOutput(true); - - const graph = session.compile(); - - expect(graph.operations[0].queries).deep.equals([finalizedDatabase]); - }); - }); - - test(`database fails when undefined`, () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - - session.operate("operate-1", ctx => ctx.database()).hasOutput(true); - - const graph = session.compile(); - - expect(graph.graphErrors.compilationErrors[0].message).deep.equals( - "Warehouse does not support multiple databases" - ); - expect(JSON.stringify(graph.operations[0].queries)).deep.equals('[""]'); - }); -}); - -suite("graph", () => { - test("circular_dependencies", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("a").dependencies("b"); - session.publish("b").dependencies("a"); - const cGraph = session.compile(); - expect( - cGraph.graphErrors.compilationErrors.filter(item => item.message.match(/Circular dependency/)) - .length - ).greaterThan(0); - }); - - test("missing_dependency", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("a", ctx => `select * from ${ctx.ref("b")}`); - const cGraph = session.compile(); - expect( - cGraph.graphErrors.compilationErrors.filter(item => item.message.match(/Missing dependency/)) - .length - ).greaterThan(0); - }); - - test("duplicate_action_names", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("a").dependencies("b"); - session.publish("b"); - session.publish("a"); - const cGraph = session.compile(); - expect( - cGraph.graphErrors.compilationErrors.filter(item => - item.message.match(/Duplicate action name/) - ).length - ).equals(2); - }); - - test("duplicate actions in compiled graph", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("a"); - session.publish("a"); - session.publish("b"); // unique action - session.publish("c"); - - session.operate("a"); - session.operate("d"); // unique action - session.operate("e"); // unique action - - session.declare({ name: "a" }); - session.declare({ name: "f" }); // unique action - session.declare({ name: "g" }); - - session.assert("c"); - session.assert("g"); - - const cGraph = session.compile(); - - expect( - [].concat(cGraph.tables, cGraph.assertions, cGraph.operations, cGraph.declarations).length - ).equals(4); - }); - - test("same action names in different schemas (ambiguity)", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("a", { schema: "foo" }); - session.publish("a", { schema: "bar" }); - session.publish("b", { schema: "foo" }).dependencies("a"); - const cGraph = session.compile(); - expect( - cGraph.graphErrors.compilationErrors.filter(item => - item.message.match( - /Ambiguous Action name: {\"name\":\"a\",\"includeDependentAssertions\":false}. Did you mean one of: foo.a, bar.a./ - ) - ).length - ).greaterThan(0); - }); - - test("same action name in same schema", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("a", { schema: "schema2" }).dependencies("b"); - session.publish("a", { schema: "schema2" }); - session.publish("b"); - const cGraph = session.compile(); - expect( - cGraph.graphErrors.compilationErrors.filter(item => - item.message.match(/Duplicate action name detected. Names within a schema must be unique/) - ).length - ).equals(2); - }); - - test("same action names in different schemas", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("b"); - session.publish("a", { schema: "schema1" }).dependencies("b"); - session.publish("a", { schema: "schema2" }); - const cGraph = session.compile(); - expect(cGraph.graphErrors.compilationErrors).deep.equals([]); - }); - - test("semi-colons at the end of files throw", () => { - // If this didn't happen, then the generated SQL could be incorrect - // because of being broken up by semi-colons. - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("a", "select 1 as x;\n"); - session.assert("b", "select 1 as x;"); - const graph = session.compile(); - expect(graph.graphErrors.compilationErrors.map(error => error.message)).deep.equals([ - "Semi-colons are not allowed at the end of SQL statements.", - "Semi-colons are not allowed at the end of SQL statements." - ]); - }); - - test("defaultLocation must be set in BigQuery", () => { - const session = new Session( - path.dirname(__filename), - dataform.ProjectConfig.create({ - warehouse: "bigquery", - defaultSchema: "schema" - }) - ); - const graph = session.compile(); - expect(graph.graphErrors.compilationErrors.map(error => error.message)).deep.equals([ - "A defaultLocation is required for BigQuery. This can be configured in workflow_settings.yaml." - ]); - }); }); From 3a81f626b0852f58632c207f43badd7ae1cb88e3 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Thu, 2 Jan 2025 11:32:05 +0000 Subject: [PATCH 2/4] More progress with conversion --- core/main_test.ts | 301 ++++++++++++++++++++++++++++++++++++++++ tests/core/core.spec.ts | 61 -------- 2 files changed, 301 insertions(+), 61 deletions(-) diff --git a/core/main_test.ts b/core/main_test.ts index 8b9d1d772..65bde587f 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -2488,6 +2488,43 @@ SELECT 2` }); }); }); + + // These tests are here for legacy purposes, and should be considered for deletion once all SQLX + // options are strongly typed by protobuf definitions. + [ + { + testName: "partitions invalid for BigQuery views", + sqlxConfigBlock: { + type: "view", + bigquery: { + partitionBy: "some_partition" + } + }, + expectedError: + "partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views" + } + ].forEach(testParameters => { + test(testParameters.testName, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); + const file = `config ${testParameters.sqlxConfigBlock}`; + console.log("🚀 ~ test ~ file:", file); + fs.writeFileSync(path.join(projectDir, `definitions/file.sqlx`), file); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.operations)).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.declarations)).deep.equals([]); + }); + }); }); suite("action config options", () => { @@ -3707,6 +3744,270 @@ publish("name", { } ); }); + + test("ref resolved correctly", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/operation.sqlx"), + ` +config { + hasOutput: true +} +SELECT 1` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/publish.js"), + ` +publish("name", { + type: "${tableType}", +}).query(ctx => \`SELECT * FROM \${ctx.ref('operation')}\`)` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + canonicalTarget: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + }, + dependencyTargets: [ + { + database: "defaultProject", + name: "operation", + schema: "defaultDataset" + } + ], + disabled: false, + enumType: tableType.toUpperCase(), + fileName: "definitions/publish.js", + query: "SELECT * FROM `defaultProject.defaultDataset.operation`", + target: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + }, + type: tableType, + ...(tableType === "incremental" + ? { + incrementalQuery: "SELECT * FROM `defaultProject.defaultDataset.operation`", + protected: true + } + : {}) + } + ]); + }); + }); + }); + + suite("operate", () => { + [ + TestConfigs.bigqueryWithDefaultProjectAndDataset, + { ...TestConfigs.bigqueryWithDatasetSuffix, defaultProject: "defaultProject" }, + { ...TestConfigs.bigqueryWithNamePrefix, defaultProject: "defaultProject" } + ].forEach(projectConfig => { + test( + `operate with project suffix ` + + `'${projectConfig.projectSuffix}', dataset suffix ` + + `'${projectConfig.datasetSuffix}', and name prefix '${projectConfig.namePrefix}'`, + () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + dumpYaml(dataform.WorkflowSettings.create(projectConfig)) + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/operate.js"), + ` +operate("name", { + type: "operations", +}).queries(_ => ["SELECT 1", "SELECT 2"])` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.operations)).deep.equals( + asPlainObject([ + { + target: { + database: projectConfig.projectSuffix + ? `${projectConfig.defaultProject}_${projectConfig.projectSuffix}` + : projectConfig.defaultProject, + schema: projectConfig.datasetSuffix + ? `${projectConfig.defaultDataset}_${projectConfig.datasetSuffix}` + : projectConfig.defaultDataset, + name: projectConfig.namePrefix ? `${projectConfig.namePrefix}_name` : "name" + }, + canonicalTarget: { + database: projectConfig.defaultProject, + schema: projectConfig.defaultDataset, + name: "name" + }, + fileName: "definitions/operate.js", + queries: ["SELECT 1", "SELECT 2"] + } + ]) + ); + } + ); + }); + + test("ref resolved correctly", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/table.sqlx"), + `config {type: "table"} SELECT 1` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/operate.js"), + ` +operate("name", { + type: "operations", +}).queries(ctx => [\`SELECT * FROM \${ctx.ref('table')}\`])` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.operations)).deep.equals([ + { + canonicalTarget: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + }, + dependencyTargets: [ + { + database: "defaultProject", + name: "table", + schema: "defaultDataset" + } + ], + fileName: "definitions/operate.js", + queries: ["SELECT * FROM `defaultProject.defaultDataset.table`"], + target: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + } + } + ]); + }); + }); + + suite("assert", () => { + [ + TestConfigs.bigqueryWithDefaultProjectAndDataset, + { ...TestConfigs.bigqueryWithDatasetSuffix, defaultProject: "defaultProject" }, + { ...TestConfigs.bigqueryWithNamePrefix, defaultProject: "defaultProject" } + ].forEach(projectConfig => { + test( + `assert with project suffix ` + + `'${projectConfig.projectSuffix}', dataset suffix ` + + `'${projectConfig.datasetSuffix}', and name prefix '${projectConfig.namePrefix}'`, + () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + dumpYaml(dataform.WorkflowSettings.create(projectConfig)) + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/assert.js"), + ` +assert("name", { + type: "operations", +}).query(_ => "SELECT 1")` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals( + asPlainObject([ + { + target: { + database: projectConfig.projectSuffix + ? `${projectConfig.defaultProject}_${projectConfig.projectSuffix}` + : projectConfig.defaultProject, + schema: projectConfig.datasetSuffix + ? `${projectConfig.defaultDataset}_${projectConfig.datasetSuffix}` + : projectConfig.defaultDataset, + name: projectConfig.namePrefix ? `${projectConfig.namePrefix}_name` : "name" + }, + canonicalTarget: { + database: projectConfig.defaultProject, + schema: projectConfig.defaultDataset, + name: "name" + }, + fileName: "definitions/assert.js", + query: "SELECT 1" + } + ]) + ); + } + ); + }); + + test("ref resolved correctly", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/table.sqlx"), + `config {type: "table"} SELECT 1` + ); + fs.writeFileSync( + path.join(projectDir, "definitions/assert.js"), + ` +assert("name", { + type: "assert", +}).query(ctx => \`SELECT * FROM \${ctx.ref('table')}\`)` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals([ + { + canonicalTarget: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + }, + dependencyTargets: [ + { + database: "defaultProject", + name: "table", + schema: "defaultDataset" + } + ], + fileName: "definitions/assert.js", + query: "SELECT * FROM `defaultProject.defaultDataset.table`", + target: { + database: "defaultProject", + name: "name", + schema: "defaultDataset" + } + } + ]); }); }); }); diff --git a/tests/core/core.spec.ts b/tests/core/core.spec.ts index fd3afdb34..71adfa9ac 100644 --- a/tests/core/core.spec.ts +++ b/tests/core/core.spec.ts @@ -208,65 +208,4 @@ suite("@dataform/core", () => { ); expect(graph.graphErrors.compilationErrors).to.deep.equals([]); }); - - test(`database fails when undefined`, () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("test", { type: "table" }).query(ctx => ctx.database()); - - const graph = session.compile(); - - const testTable = graph.tables.find( - table => targetAsReadableString(table.target) === "schema.test" - ); - - expect(graph.graphErrors.compilationErrors[0].message).deep.equals( - "Warehouse does not support multiple databases" - ); - expect(testTable.query).deep.equals(""); - }); -}); - -suite("operate", () => { - test("ref", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.operate("operate-1", () => `select 1 as sample`).hasOutput(true); - session.operate("operate-2", ctx => `select * from ${ctx.ref("operate-1")}`).hasOutput(true); - - const graph = session.compile(); - - expect(graph.graphErrors.compilationErrors).deep.equals([]); - expect(graph) - .to.have.property("operations") - .to.be.an("array") - .to.have.lengthOf(2); - - expect(targetAsReadableString(graph.operations[0].target)).equals("schema.operate-1"); - expect( - graph.operations[0].dependencyTargets.map(dependency => targetAsReadableString(dependency)) - ).deep.equals([]); - expect(graph.operations[0].queries).deep.equals(["select 1 as sample"]); - - expect(targetAsReadableString(graph.operations[1].target)).equals("schema.operate-2"); - expect( - graph.operations[1].dependencyTargets.map(dependency => targetAsReadableString(dependency)) - ).deep.equals(["schema.operate-1"]); - expect(graph.operations[1].queries).deep.equals(["select * from `schema.operate-1`"]); - }); - - [ - { testConfig: TestConfigs.bigquery, finalizedName: "operate-1" }, - { - testConfig: TestConfigs.bigqueryWithTablePrefix, - finalizedName: "prefix_operate-1" - } - ].forEach(({ testConfig, finalizedName }) => { - test(`name with prefix: "${finalizedName}"`, () => { - const session = new Session(path.dirname(__filename), testConfig); - session.operate("operate-1", ctx => ctx.name()).hasOutput(true); - - const graph = session.compile(); - - expect(graph.operations[0].queries).deep.equals([finalizedName]); - }); - }); }); From 8c643ba65c96bf30908f262690502592312ec093 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Thu, 2 Jan 2025 14:02:12 +0000 Subject: [PATCH 3/4] Migrate the remaining invalid publish errors --- core/main_test.ts | 197 ++++++++++++++++++++++++++++++------- tests/core/core.spec.ts | 211 ---------------------------------------- 2 files changed, 160 insertions(+), 248 deletions(-) delete mode 100644 tests/core/core.spec.ts diff --git a/core/main_test.ts b/core/main_test.ts index 65bde587f..026d0ca51 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -2488,43 +2488,6 @@ SELECT 2` }); }); }); - - // These tests are here for legacy purposes, and should be considered for deletion once all SQLX - // options are strongly typed by protobuf definitions. - [ - { - testName: "partitions invalid for BigQuery views", - sqlxConfigBlock: { - type: "view", - bigquery: { - partitionBy: "some_partition" - } - }, - expectedError: - "partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views" - } - ].forEach(testParameters => { - test(testParameters.testName, () => { - const projectDir = tmpDirFixture.createNewTmpDir(); - fs.writeFileSync( - path.join(projectDir, "workflow_settings.yaml"), - VALID_WORKFLOW_SETTINGS_YAML - ); - fs.mkdirSync(path.join(projectDir, "definitions")); - fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); - const file = `config ${testParameters.sqlxConfigBlock}`; - console.log("🚀 ~ test ~ file:", file); - fs.writeFileSync(path.join(projectDir, `definitions/file.sqlx`), file); - - const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); - - expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.operations)).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals([]); - expect(asPlainObject(result.compile.compiledGraph.declarations)).deep.equals([]); - }); - }); }); suite("action config options", () => { @@ -4010,6 +3973,166 @@ assert("name", { ]); }); }); + + suite("invalid options", () => { + // These tests are here for legacy purposes, and should be considered for deletion once all + // config options are strongly typed by protobuf definitions. + // TODO(ekrekr): these should also be tested as SQLX configs, but the errors for those seem to + // have been broken at some point - the previous tests only covered the JS API. + [ + { + testName: "partitionBy invalid for BigQuery views", + fileContents: ` +publish("name", { + type: "view", + bigquery: { + partitionBy: "some_partition" + } +})`, + expectedError: + "partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views" + }, + { + testName: "clusterBy invalid for BigQuery views", + fileContents: ` +publish("name", { + type: "view", + bigquery: { + clusterBy: ["some_cluster"] + } +})`, + expectedError: + "partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views" + }, + { + testName: "partitionExpirationDays invalid for BigQuery views", + fileContents: ` +publish("name", { + type: "view", + bigquery: { + partitionExpirationDays: 7 + } +})`, + expectedError: + "partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views" + }, + { + testName: "requirePartitionFilter invalid for BigQuery views", + fileContents: ` +publish("name", { + type: "view", + bigquery: { + requirePartitionFilter: true + } +})`, + expectedError: + "partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views" + }, + { + testName: "partitionExpirationDays invalid for BigQuery materialized views", + fileContents: ` +publish("name", { + type: "view", + materialized: true, + bigquery: { + partitionBy: "some_partition", + clusterBy: ["some_cluster"], + partitionExpirationDays: 7 + } +})`, + expectedError: + "requirePartitionFilter/partitionExpirationDays are not valid for BigQuery materialized views" + }, + { + testName: "requirePartitionFilter invalid for BigQuery materialized views", + fileContents: ` +publish("name", { + type: "view", + materialized: true, + bigquery: { + partitionBy: "some_partition", + clusterBy: ["some_cluster"], + requirePartitionFilter: true + } +})`, + expectedError: + "requirePartitionFilter/partitionExpirationDays are not valid for BigQuery materialized views" + }, + { + testName: "materialize invalid for BigQuery tables", + fileContents: ` +publish("name", { + type: "table", + materialized: true, +})`, + expectedError: "The 'materialized' option is only valid for BigQuery views" + }, + { + testName: "partitionExpirationDays invalid for BigQuery tables", + fileContents: ` +publish("name", { + type: "table", + bigquery: { + partitionExpirationDays: 7 + } +})`, + expectedError: + "requirePartitionFilter/partitionExpirationDays are not valid for non partitioned BigQuery tables" + }, + { + testName: "duplicate partitionExpirationDays is invalid", + fileContents: ` +publish("name", { + type: "table", + bigquery: { + partitionBy: "partition", + partitionExpirationDays: 1, + additionalOptions: { + partition_expiration_days: "7" + } + } +})`, + expectedError: "partitionExpirationDays has been declared twice" + }, + { + testName: "duplicate requirePartitionFilter is invalid", + fileContents: ` +publish("name", { + type: "table", + bigquery: { + partitionBy: "partition", + requirePartitionFilter: true, + additionalOptions: { + require_partition_filter: "false" + } + } +})`, + expectedError: "requirePartitionFilter has been declared twice" + } + ].forEach(testParameters => { + test(testParameters.testName, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1"); + fs.writeFileSync( + path.join(projectDir, `definitions/file.js`), + testParameters.fileContents + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect( + result.compile.compiledGraph.graphErrors.compilationErrors.map( + compilationError => compilationError.message + ) + ).deep.equals([testParameters.expectedError]); + }); + }); + }); }); }); diff --git a/tests/core/core.spec.ts b/tests/core/core.spec.ts deleted file mode 100644 index 71adfa9ac..000000000 --- a/tests/core/core.spec.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { expect } from "chai"; -import * as path from "path"; - -import { Session } from "df/core/session"; -import { targetAsReadableString } from "df/core/targets"; -import { dataform } from "df/protos/ts"; -import { suite, test } from "df/testing"; - -// TODO(ekrekr): migrate the tests in this file to core/main_test.ts. - -class TestConfigs { - public static bigquery = dataform.ProjectConfig.create({ - warehouse: "bigquery", - defaultSchema: "schema", - defaultLocation: "US" - }); - - public static bigqueryWithDefaultDatabase = dataform.ProjectConfig.create({ - ...TestConfigs.bigquery, - defaultDatabase: "default-database" - }); - - public static bigqueryWithSchemaSuffix = dataform.ProjectConfig.create({ - ...TestConfigs.bigquery, - schemaSuffix: "suffix" - }); - - public static bigqueryWithDefaultDatabaseAndSuffix = dataform.ProjectConfig.create({ - ...TestConfigs.bigqueryWithDefaultDatabase, - databaseSuffix: "suffix" - }); - - public static bigqueryWithTablePrefix = dataform.ProjectConfig.create({ - ...TestConfigs.bigquery, - tablePrefix: "prefix" - }); -} - -suite("@dataform/core", () => { - test("validation_bigquery_fail", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("example_partitionBy_view_fail", { - type: "view", - bigquery: { - partitionBy: "some_partition" - } - }); - session.publish("example_clusterBy_view_fail", { - type: "view", - bigquery: { - clusterBy: ["some_cluster"] - } - }); - session.publish("example_expiring_view_fail", { - type: "view", - bigquery: { - partitionExpirationDays: 7 - } - }); - session.publish("example_require_partition_filter_view_fail", { - type: "view", - bigquery: { - requirePartitionFilter: true - } - }); - session.publish("example_expiring_materialized_view_fail", { - type: "view", - materialized: true, - bigquery: { - partitionBy: "some_partition", - clusterBy: ["some_cluster"], - partitionExpirationDays: 7 - } - }); - session.publish("example_require_partition_filter_materialized_view_fail", { - type: "view", - materialized: true, - bigquery: { - partitionBy: "some_partition", - clusterBy: ["some_cluster"], - requirePartitionFilter: true - } - }); - session.publish("example_materialize_table_fail", { - type: "table", - materialized: true - }); - session.publish("example_expiring_non_partitioned_fail", { - type: "table", - bigquery: { - partitionExpirationDays: 7 - } - }); - session.publish("example_duplicate_partition_expiration_days_fail", { - type: "table", - bigquery: { - partitionBy: "partition", - partitionExpirationDays: 1, - additionalOptions: { - partition_expiration_days: "7" - } - } - }); - session.publish("example_duplicate_require_partition_filter_fail", { - type: "table", - bigquery: { - partitionBy: "partition", - requirePartitionFilter: true, - additionalOptions: { - require_partition_filter: "false" - } - } - }); - - const graph = session.compile(); - - expect( - graph.graphErrors.compilationErrors.map(({ message, actionName }) => ({ - message, - actionName - })) - ).has.deep.members([ - { - actionName: "schema.example_partitionBy_view_fail", - message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` - }, - { - actionName: "schema.example_clusterBy_view_fail", - message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` - }, - { - actionName: "schema.example_expiring_view_fail", - message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` - }, - { - actionName: "schema.example_require_partition_filter_view_fail", - message: `partitionBy/clusterBy/requirePartitionFilter/partitionExpirationDays are not valid for BigQuery views` - }, - { - actionName: "schema.example_expiring_materialized_view_fail", - message: `requirePartitionFilter/partitionExpirationDays are not valid for BigQuery materialized views` - }, - { - actionName: "schema.example_require_partition_filter_materialized_view_fail", - message: `requirePartitionFilter/partitionExpirationDays are not valid for BigQuery materialized views` - }, - { - actionName: "schema.example_materialize_table_fail", - message: "The 'materialized' option is only valid for BigQuery views" - }, - { - actionName: "schema.example_expiring_non_partitioned_fail", - message: - "requirePartitionFilter/partitionExpirationDays are not valid for non partitioned BigQuery tables" - }, - { - actionName: "schema.example_duplicate_partition_expiration_days_fail", - message: "partitionExpirationDays has been declared twice" - }, - { - actionName: "schema.example_duplicate_require_partition_filter_fail", - message: "requirePartitionFilter has been declared twice" - } - ]); - }); - - test("validation_bigquery_pass", () => { - const session = new Session(path.dirname(__filename), TestConfigs.bigquery); - session.publish("example_partitionBy_view_fail", { - type: "table", - bigquery: { - partitionBy: "some_partition", - clusterBy: ["some_column", "some_other_column"], - partitionExpirationDays: 7, - requirePartitionFilter: false - } - }); - session.publish("example_materialized_view", { - type: "view", - materialized: true - }); - session.publish("example_additional_options", { - type: "table", - bigquery: { - additionalOptions: { - friendlyName: "name" - } - } - }); - - const graph = session.compile(); - - expect(graph.tables[0].bigquery).to.deep.equals( - dataform.BigQueryOptions.create({ - clusterBy: ["some_column", "some_other_column"], - partitionBy: "some_partition", - partitionExpirationDays: 7, - requirePartitionFilter: false - }) - ); - expect(graph.tables[1].materialized).to.equals(true); - expect(graph.tables[2].bigquery).to.deep.equals( - dataform.BigQueryOptions.create({ - additionalOptions: { - friendlyName: "name" - } - }) - ); - expect(graph.graphErrors.compilationErrors).to.deep.equals([]); - }); -}); From 23810c8b16295adb0c0bba0bf7c5cc388a3db17d Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Thu, 2 Jan 2025 14:05:28 +0000 Subject: [PATCH 4/4] Tidy --- core/main_test.ts | 6 +----- tests/core/BUILD | 15 --------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 tests/core/BUILD diff --git a/core/main_test.ts b/core/main_test.ts index 026d0ca51..a60b2bc86 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -357,7 +357,7 @@ publish("b", "SELECT 1;");` const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); - expect(asPlainObject(getActionsFromResult(tableType, result)[0]?.disabled)).is.true; + expect(asPlainObject(getActionsFromResult(tableType, result)[0]?.disabled)).equals(true); }); test(`${tableType} target can be overridden by project config override`, () => { @@ -3975,10 +3975,6 @@ assert("name", { }); suite("invalid options", () => { - // These tests are here for legacy purposes, and should be considered for deletion once all - // config options are strongly typed by protobuf definitions. - // TODO(ekrekr): these should also be tested as SQLX configs, but the errors for those seem to - // have been broken at some point - the previous tests only covered the JS API. [ { testName: "partitionBy invalid for BigQuery views", diff --git a/tests/core/BUILD b/tests/core/BUILD deleted file mode 100644 index 7e28d212d..000000000 --- a/tests/core/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -load("//testing:index.bzl", "ts_test_suite") - -ts_test_suite( - name = "tests", - srcs = glob(["**/*.ts"]), - deps = [ - "//core", - "//protos:ts", - "//testing", - "@npm//@types/chai", - "@npm//@types/long", - "@npm//@types/node", - "@npm//chai", - ], -)