From c617717d0e658ada936a42f1bee69f03e7c22fe4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 24 Mar 2025 17:50:53 +0200 Subject: [PATCH 01/22] some improvements in transformYamlCubeObj() --- .../src/compiler/YamlCompiler.ts | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 30933af64df3d..322cdc84f1aa4 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -127,12 +127,29 @@ export class YamlCompiler { private transformYamlCubeObj(cubeObj, errorsReport: ErrorReporter) { camelizeCube(cubeObj); - cubeObj.measures = this.yamlArrayToObj(cubeObj.measures || [], 'measure', errorsReport); - cubeObj.dimensions = this.yamlArrayToObj(cubeObj.dimensions || [], 'dimension', errorsReport); - cubeObj.segments = this.yamlArrayToObj(cubeObj.segments || [], 'segment', errorsReport); - cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations || [], 'preAggregation', errorsReport); - cubeObj.joins = this.yamlArrayToObj(cubeObj.joins || [], 'join', errorsReport); - cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies || [], 'hierarchies', errorsReport); + if (cubeObj.measures) { + cubeObj.measures = this.yamlArrayToObj(cubeObj.measures, 'measure', errorsReport); + } + + if (cubeObj.dimensions) { + cubeObj.dimensions = this.yamlArrayToObj(cubeObj.dimensions, 'dimension', errorsReport); + } + + if (cubeObj.segments) { + cubeObj.segments = this.yamlArrayToObj(cubeObj.segments, 'segment', errorsReport); + } + + if (cubeObj.preAggregations) { + cubeObj.preAggregations = this.yamlArrayToObj(cubeObj.preAggregations, 'preAggregation', errorsReport); + } + + if (cubeObj.joins) { + cubeObj.joins = this.yamlArrayToObj(cubeObj.joins, 'join', errorsReport); + } + + if (cubeObj.hierarchies) { + cubeObj.hierarchies = this.yamlArrayToObj(cubeObj.hierarchies, 'hierarchies', errorsReport); + } return this.transpileYaml(cubeObj, [], cubeObj.name, errorsReport); } From 3c5b81bd16785f452def599766ba30ab8a55bcc5 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 24 Mar 2025 18:58:00 +0200 Subject: [PATCH 02/22] implement inheritance for preaggs & DAPs --- .../src/compiler/CubeSymbols.ts | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index fec0159c8cb99..cced8c1e12037 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -18,7 +18,7 @@ interface CubeDefinition { hierarchies?: Record; preAggregations?: Record; joins?: Record; - accessPolicy?: Record; + accessPolicy?: any[]; includes?: any; excludes?: any; cubes?: any; @@ -34,7 +34,7 @@ interface SplitViews { const FunctionRegex = /function\s+\w+\(([A-Za-z0-9_,]*)|\(([\s\S]*?)\)\s*=>|\(?(\w+)\)?\s*=>/; export const CONTEXT_SYMBOLS = { SECURITY_CONTEXT: 'securityContext', - // SECURITY_CONTEXT has been deprecated, however security_context (lowecase) + // SECURITY_CONTEXT has been deprecated, however security_context (lowercase) // is allowed in RBAC policies for query-time attribute matching security_context: 'securityContext', securityContext: 'securityContext', @@ -103,10 +103,13 @@ export class CubeSymbols { } public createCube(cubeDefinition: CubeDefinition) { + let preAggregations: any; + let joins: any; let measures: any; let dimensions: any; let segments: any; let hierarchies: any; + let accessPolicy: any; const cubeObject = Object.assign({ allDefinitions(type: string) { @@ -119,6 +122,27 @@ export class CubeSymbols { return { ...cubeDefinition[type] }; } }, + + get preAggregations() { + if (!preAggregations) { + preAggregations = this.allDefinitions('preAggregations'); + } + return preAggregations; + }, + set preAggregations(v) { + // Dont allow to modify + }, + + get joins() { + if (!joins) { + joins = this.allDefinitions('joins'); + } + return joins; + }, + set joins(v) { + // Dont allow to modify + }, + get measures() { if (!measures) { measures = this.allDefinitions('measures'); @@ -156,7 +180,23 @@ export class CubeSymbols { return hierarchies; }, set hierarchies(v) { - // + // Dont allow to modify + }, + + get accessPolicy() { + if (!accessPolicy) { + const parentAcls = cubeDefinition.extends ? super.accessPolicy : []; + accessPolicy = [...(parentAcls || []), ...(cubeDefinition.accessPolicy || [])]; + } + // Schema validator expects accessPolicy to be not empty if defined + if (accessPolicy.length) { + return accessPolicy; + } else { + return undefined; + } + }, + set accessPolicy(v) { + // Dont allow to modify } }, cubeDefinition); From 1952142b6085a029d3a19324bc6263f728cbd636 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 24 Mar 2025 19:31:33 +0200 Subject: [PATCH 03/22] =?UTF-8?q?rename=20prepareCompiler=20=E2=86=92=20pr?= =?UTF-8?q?epareJsCompiler()=20in=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clickhouse-dataschema-compiler.test.ts | 20 ++--- .../clickhouse-graph-builder.test.ts | 4 +- .../mssql/mssql-cumulative-measures.test.ts | 10 +-- .../mssql/mssql-pre-aggregations.test.ts | 6 +- .../integration/mssql/mssql-ungrouped.test.ts | 4 +- .../mysql/mysql-pre-aggregations.test.ts | 12 +-- .../integration/postgres/async-module.test.ts | 6 +- .../integration/postgres/cube-views.test.ts | 4 +- .../postgres/dataschema-compiler.test.ts | 34 ++++---- .../member-expressions-on-views.test.ts | 4 +- .../postgres/multi-fact-join.test.ts | 4 +- .../postgres/pre-agg-allow-non-strict.test.ts | 16 ++-- .../postgres/pre-aggregations-alias.test.ts | 82 +++++++++---------- .../postgres/pre-aggregations-time.test.ts | 8 +- .../postgres/pre-aggregations.test.ts | 4 +- .../postgres/sql-generation-logic.test.ts | 6 +- .../postgres/sql-generation.test.ts | 6 +- .../postgres/sub-query-dimensions.test.ts | 8 +- .../postgres/views-join-order-2.test.ts | 4 +- .../postgres/views-join-order.test.ts | 4 +- .../test/integration/postgres/views.test.ts | 4 +- .../postgres/yaml-compiler.test.ts | 5 +- ...{PrepareCompiler.js => PrepareCompiler.ts} | 18 +++- .../test/unit/base-query.test.ts | 26 +++--- .../test/unit/extensions.test.ts | 6 +- .../test/unit/hierarchies.test.ts | 4 +- .../test/unit/mongobi-query.test.ts | 4 +- .../test/unit/mssql-query.test.ts | 20 ++--- .../test/unit/postgres-query.test.ts | 4 +- .../test/unit/pre-aggregations.test.ts | 6 +- .../test/unit/schema.test.ts | 18 ++-- .../test/unit/transpile-speed.test.ts | 4 +- .../test/unit/transpilers.test.ts | 6 +- 33 files changed, 190 insertions(+), 181 deletions(-) rename packages/cubejs-schema-compiler/test/unit/{PrepareCompiler.js => PrepareCompiler.ts} (54%) diff --git a/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-dataschema-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-dataschema-compiler.test.ts index 1d2cc00e649df..8ed3ab6ebfe35 100644 --- a/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-dataschema-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-dataschema-compiler.test.ts @@ -2,7 +2,7 @@ import { CompileError } from '../../../src/compiler/CompileError'; import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery'; import { prepareCompiler } from '../../../src/compiler/PrepareCompiler'; -import { prepareCompiler as testPrepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { ClickHouseDbRunner } from './ClickHouseDbRunner'; import { logSqlAndParams } from '../../unit/TestUtil'; @@ -18,7 +18,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); it('gutter', () => { - const { compiler } = testPrepareCompiler(` + const { compiler } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -53,7 +53,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); it('error', () => { - const { compiler } = testPrepareCompiler(` + const { compiler } = prepareJsCompiler(` cube({}, { measures: {} }) @@ -69,7 +69,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); it('duplicate member', () => { - const { compiler } = testPrepareCompiler(` + const { compiler } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -104,7 +104,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); it('calculated metrics', () => { - const { compiler, cubeEvaluator, joinGraph } = testPrepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -178,7 +178,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); it('static dimension case', async () => { - const { compiler, cubeEvaluator, joinGraph } = testPrepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -236,7 +236,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); it('dynamic dimension case', () => { - const { compiler, cubeEvaluator, joinGraph } = testPrepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -310,7 +310,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); { - const { compiler, cubeEvaluator, joinGraph } = testPrepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -418,7 +418,7 @@ describe('ClickHouse DataSchemaCompiler', () => { }); it('contexts', () => { - const { compiler, contextEvaluator } = testPrepareCompiler(` + const { compiler, contextEvaluator } = prepareJsCompiler(` cube('Visitors', { sql: \` select * from visitors @@ -454,7 +454,7 @@ describe('ClickHouse DataSchemaCompiler', () => { dbRunner.supportsExtendedDateTimeResults, 'handles dates before 1970 correctly for time dimensions', async () => { - const { compiler, cubeEvaluator, joinGraph } = testPrepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('Events', { sql: \` select * from events diff --git a/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-graph-builder.test.ts b/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-graph-builder.test.ts index 599d764f0cdbf..8d9b75e08fb8a 100644 --- a/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-graph-builder.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/clickhouse/clickhouse-graph-builder.test.ts @@ -1,5 +1,5 @@ import { UserError } from '../../../src/compiler/UserError'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { ClickHouseDbRunner } from './ClickHouseDbRunner'; import { debugLog, logSqlAndParams } from '../../unit/TestUtil'; import { ClickHouseQuery } from '../../../src/adapter/ClickHouseQuery'; @@ -13,7 +13,7 @@ describe('ClickHouse JoinGraph', () => { await dbRunner.tearDown(); }); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` const perVisitorRevenueMeasure = { type: 'number', sql: new Function('visitor_revenue', 'visitor_count', 'return visitor_revenue + "/" + visitor_count') diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts index 132346395100d..e65d7a44084b1 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-cumulative-measures.test.ts @@ -1,5 +1,5 @@ import { MssqlQuery } from '../../../src/adapter/MssqlQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { MSSqlDbRunner } from './MSSqlDbRunner'; describe('MSSqlCumulativeMeasures', () => { @@ -11,19 +11,19 @@ describe('MSSqlCumulativeMeasures', () => { await dbRunner.tearDown(); }); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from ##visitors \`, - + joins: {}, measures: { count: { type: 'count' }, - + unboundedCount: { type: 'count', rollingWindow: { @@ -47,7 +47,7 @@ describe('MSSqlCumulativeMeasures', () => { sql: 'created_at' }, }, - + preAggregations: {} }) `); diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts index 3cdfc8539567d..949e881692296 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-pre-aggregations.test.ts @@ -1,6 +1,6 @@ import R from 'ramda'; import { MssqlQuery } from '../../../src/adapter/MssqlQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { MSSqlDbRunner } from './MSSqlDbRunner'; import { createJoinedCubesSchema } from '../../unit/utils'; @@ -13,7 +13,7 @@ describe('MSSqlPreAggregations', () => { await dbRunner.tearDown(); }); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from ##visitors @@ -175,7 +175,7 @@ describe('MSSqlPreAggregations', () => { }) `); - const joinedSchemaCompilers = prepareCompiler(createJoinedCubesSchema()); + const joinedSchemaCompilers = prepareJsCompiler(createJoinedCubesSchema()); function replaceTableName(query, preAggregation, suffix) { const [toReplace, params] = query; diff --git a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts index 9aab36c0cfcb8..7c19c7e8767ba 100644 --- a/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mssql/mssql-ungrouped.test.ts @@ -1,5 +1,5 @@ import { MssqlQuery } from '../../../src/adapter/MssqlQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { MSSqlDbRunner } from './MSSqlDbRunner'; describe('MSSqlUngrouped', () => { @@ -11,7 +11,7 @@ describe('MSSqlUngrouped', () => { await dbRunner.tearDown(); }); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` const perVisitorRevenueMeasure = { type: 'number', sql: new Function('visitor_revenue', 'visitor_count', 'return visitor_revenue + "/" + visitor_count') diff --git a/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts index 1ce395d6b9cf8..27f095b45167f 100644 --- a/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/mysql/mysql-pre-aggregations.test.ts @@ -1,6 +1,6 @@ import R from 'ramda'; import { MysqlQuery } from '../../../src/adapter/MysqlQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { MySqlDbRunner } from './MySqlDbRunner'; describe('MySqlPreAggregations', () => { @@ -12,7 +12,7 @@ describe('MySqlPreAggregations', () => { await dbRunner.tearDown(); }); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from visitors @@ -22,12 +22,12 @@ describe('MySqlPreAggregations', () => { count: { type: 'count' }, - + uniqueSourceCount: { sql: 'source', type: 'countDistinct' }, - + countDistinctApprox: { sql: 'id', type: 'countDistinctApprox' @@ -49,13 +49,13 @@ describe('MySqlPreAggregations', () => { sql: 'created_at' } }, - + segments: { google: { sql: \`source = 'google'\` } }, - + preAggregations: { partitioned: { type: 'rollup', diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/async-module.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/async-module.test.ts index cc962b56fb6a3..a0dd798764d3f 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/async-module.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/async-module.test.ts @@ -1,12 +1,12 @@ import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('AsyncModule', () => { jest.setTimeout(200000); it('gutter', async () => { - const { joinGraph, cubeEvaluator, compiler } = prepareCompiler(` + const { joinGraph, cubeEvaluator, compiler } = prepareJsCompiler(` const fetch = require('node-fetch'); asyncModule(async () => { @@ -57,7 +57,7 @@ describe('AsyncModule', () => { }); it('import local node module', async () => { - const { joinGraph, cubeEvaluator, compiler } = prepareCompiler(` + const { joinGraph, cubeEvaluator, compiler } = prepareJsCompiler(` import { foo } from '../../test/unit/TestHelperForImport.js'; cube(foo(), { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts index 81a70b84baa94..d46c69047b616 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/cube-views.test.ts @@ -1,12 +1,12 @@ import { getEnv } from '@cubejs-backend/shared'; import { BaseQuery, PostgresQuery } from '../../../src/adapter'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('Cube Views', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator, metaTransformer } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator, metaTransformer } = prepareJsCompiler(` cube(\`Orders\`, { sql: \` SELECT 1 as id, 1 as product_id, 'completed' as status, '2022-01-01T00:00:00.000Z'::timestamptz as created_at diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts index 81894e6da8159..faf68af2a3008 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts @@ -1,6 +1,6 @@ import { CompileError } from '../../../src/compiler/CompileError'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { prepareCompiler as originalPrepareCompiler } from '../../../src/compiler/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; @@ -8,7 +8,7 @@ describe('DataSchemaCompiler', () => { jest.setTimeout(200000); it('gutter', async () => { - const { compiler } = prepareCompiler(` + const { compiler } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -43,7 +43,7 @@ describe('DataSchemaCompiler', () => { }); it('error', async () => { - const { compiler } = prepareCompiler(` + const { compiler } = prepareJsCompiler(` cube({}, { measures: {} }) @@ -59,7 +59,7 @@ describe('DataSchemaCompiler', () => { }); it('duplicate member', () => { - const { compiler } = prepareCompiler(` + const { compiler } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -135,14 +135,14 @@ describe('DataSchemaCompiler', () => { `; it('Should compile without error, allowJsDuplicatePropsInSchema = false, valid schema', () => { - const { compiler } = prepareCompiler(validSchema, { allowJsDuplicatePropsInSchema: false }); + const { compiler } = prepareJsCompiler(validSchema, { allowJsDuplicatePropsInSchema: false }); return compiler.compile().then(() => { compiler.throwIfAnyErrors(); }); }); it('Should throw error, allowJsDuplicatePropsInSchema = false, invalid schema', () => { - const { compiler } = prepareCompiler(invalidSchema, { allowJsDuplicatePropsInSchema: false }); + const { compiler } = prepareJsCompiler(invalidSchema, { allowJsDuplicatePropsInSchema: false }); return compiler.compile().then(() => { compiler.throwIfAnyErrors(); throw new Error(); @@ -153,7 +153,7 @@ describe('DataSchemaCompiler', () => { }); it('Should compile without error, allowJsDuplicatePropsInSchema = true, invalid schema', () => { - const { compiler } = prepareCompiler(invalidSchema, { allowJsDuplicatePropsInSchema: true }); + const { compiler } = prepareJsCompiler(invalidSchema, { allowJsDuplicatePropsInSchema: true }); return compiler.compile().then(() => { compiler.throwIfAnyErrors(); }); @@ -189,7 +189,7 @@ describe('DataSchemaCompiler', () => { it('Should compile 200 schemas in less than 2500ms * 10', async () => { const repeats = 200; - const compilerWith = prepareCompiler(schema, { allowJsDuplicatePropsInSchema: false }); + const compilerWith = prepareJsCompiler(schema, { allowJsDuplicatePropsInSchema: false }); const start = new Date().getTime(); for (let i = 0; i < repeats; i++) { delete compilerWith.compiler.compilePromise; // Reset compile result @@ -204,7 +204,7 @@ describe('DataSchemaCompiler', () => { }); it('calculated metrics', async () => { - const { compiler, cubeEvaluator, joinGraph } = prepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -272,7 +272,7 @@ describe('DataSchemaCompiler', () => { }); it('static dimension case', async () => { - const { compiler, cubeEvaluator, joinGraph } = prepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -326,7 +326,7 @@ describe('DataSchemaCompiler', () => { }); it('filtered dates', async () => { - const { compiler, cubeEvaluator, joinGraph } = prepareCompiler(` + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler(` cube('visitors', { sql: \` select * from visitors @@ -435,7 +435,7 @@ describe('DataSchemaCompiler', () => { }); it('contexts', async () => { - const { compiler, contextEvaluator } = prepareCompiler(` + const { compiler, contextEvaluator } = prepareJsCompiler(` cube('Visitors', { sql: \` select * from visitors @@ -468,7 +468,7 @@ describe('DataSchemaCompiler', () => { }); it('views should not contain own members', () => { - const { compiler } = prepareCompiler(` + const { compiler } = prepareJsCompiler(` view('Visitors', { dimensions: { id: { @@ -488,10 +488,10 @@ describe('DataSchemaCompiler', () => { }); it('foreign cubes', () => { - const { compiler } = prepareCompiler(` + const { compiler } = prepareJsCompiler(` cube('Visitors', { sql: 'select * from visitors', - + dimensions: { foo: { type: 'number', @@ -499,10 +499,10 @@ describe('DataSchemaCompiler', () => { } } }); - + cube('Foreign', { sql: 'select * from foreign', - + dimensions: { bar: { type: 'number', diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/member-expressions-on-views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/member-expressions-on-views.test.ts index 57da96aab18c2..80f6beb4ffe3d 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/member-expressions-on-views.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/member-expressions-on-views.test.ts @@ -3,7 +3,7 @@ import { DataSchemaCompiler } from '../../../src/compiler/DataSchemaCompiler'; import { JoinGraph } from '../../../src/compiler/JoinGraph'; import { CubeEvaluator } from '../../../src/compiler/CubeEvaluator'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('Member expressions on views', () => { @@ -270,7 +270,7 @@ describe('Member expressions on views', () => { let cubeEvaluator: CubeEvaluator; beforeAll(async () => { - ({ compiler, joinGraph, cubeEvaluator } = prepareCompiler(model)); + ({ compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(model)); await compiler.compile(); }); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts index bd3f795803b38..a2eac754d94b2 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/multi-fact-join.test.ts @@ -2,13 +2,13 @@ import { getEnv, } from '@cubejs-backend/shared'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('Multi-fact join', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`orders\`, { sql: \` SELECT 79 AS id, 1 AS amount, 1 AS city_id UNION ALL diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts index 0a19233ab49cc..9c06a1b914ba1 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-agg-allow-non-strict.test.ts @@ -1,5 +1,5 @@ import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; const getSql = () => ` select 3060 as row_id, 'CA-2017-131492' as order_id, to_date('2020-10-19', 'YYYY-MM-DD') as order_date, 'HH-15010' as customer_id, 'San Francisco' as city, 'Furniture' as category, 'Tables' as sub_category, 'Anderson Hickey Conga Table Tops & Accessories' as product_name, 24.36800 as sales, 2 as quantity, 0.20000 as discount, -3.35060 as profit union all @@ -214,7 +214,7 @@ describe( jest.setTimeout(200000); const { compiler, joinGraph, cubeEvaluator } = - prepareCompiler(getCube(true, false, false)); + prepareJsCompiler(getCube(true, false, false)); it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); @@ -284,7 +284,7 @@ describe( jest.setTimeout(200000); const { compiler, joinGraph, cubeEvaluator } = - prepareCompiler(getCube(false, true, false)); + prepareJsCompiler(getCube(false, true, false)); it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); @@ -354,7 +354,7 @@ describe( jest.setTimeout(200000); const { compiler, joinGraph, cubeEvaluator } = - prepareCompiler(getCube(false, false, true)); + prepareJsCompiler(getCube(false, false, true)); it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); @@ -424,7 +424,7 @@ describe( jest.setTimeout(200000); const { compiler, joinGraph, cubeEvaluator } = - prepareCompiler(getCube(true, true, false)); + prepareJsCompiler(getCube(true, true, false)); it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); @@ -485,7 +485,7 @@ describe( jest.setTimeout(200000); const { compiler, joinGraph, cubeEvaluator } = - prepareCompiler(getCube(true, false, true)); + prepareJsCompiler(getCube(true, false, true)); it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); @@ -546,7 +546,7 @@ describe( jest.setTimeout(200000); const { compiler, joinGraph, cubeEvaluator } = - prepareCompiler(getCube(false, true, true)); + prepareJsCompiler(getCube(false, true, true)); it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); @@ -607,7 +607,7 @@ describe( jest.setTimeout(200000); const { compiler, joinGraph, cubeEvaluator } = - prepareCompiler(getCube(true, true, true)); + prepareJsCompiler(getCube(true, true, true)); it('month query with the `month` granularity match `MonthlyData`', async () => { await compiler.compile(); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-alias.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-alias.test.ts index 54b89052023a7..dcc6f97c08f92 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-alias.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-alias.test.ts @@ -1,19 +1,19 @@ import R from 'ramda'; import { UserError } from '../../../src/compiler/UserError'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('PreAggregationsAlias', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from visitors WHERE \${FILTER_PARAMS.visitors.createdAt.filter('created_at')} \`, sqlAlias: 'vis', - + joins: { visitor_checkins: { relationship: 'hasMany', @@ -21,7 +21,7 @@ describe('PreAggregationsAlias', () => { } }, - measures: { + measures: { count: { type: 'count' }, @@ -29,22 +29,22 @@ describe('PreAggregationsAlias', () => { sql: 'id', type: 'sum' }, - + checkinsTotal: { sql: \`\${checkinsCount}\`, type: 'sum' }, - + uniqueSourceCount: { sql: 'source', type: 'countDistinct' }, - + countDistinctApprox: { sql: 'id', type: 'countDistinctApprox' }, - + ratio: { sql: \`\${uniqueSourceCount} / nullif(\${checkinsTotal}, 0)\`, type: 'number' @@ -72,13 +72,13 @@ describe('PreAggregationsAlias', () => { propagateFiltersToSubQuery: true } }, - + segments: { google: { sql: \`source = 'google'\` } }, - + preAggregations: { default: { sqlAlias: 'visitors_alias_d', @@ -93,17 +93,17 @@ describe('PreAggregationsAlias', () => { }, partitionGranularity: 'day', timeDimensionReference: createdAt - }, + }, } }) - - + + cube(\`rollup_visitors\`, { sql: \` select * from visitors WHERE \${FILTER_PARAMS.visitors.createdAt.filter('created_at')} \`, sqlAlias: 'rvis', - + joins: { visitor_checkins: { relationship: 'hasMany', @@ -111,7 +111,7 @@ describe('PreAggregationsAlias', () => { } }, - measures: { + measures: { count: { type: 'count' }, @@ -119,22 +119,22 @@ describe('PreAggregationsAlias', () => { sql: 'id', type: 'sum' }, - + checkinsTotal: { sql: \`\${checkinsCount}\`, type: 'sum' }, - + uniqueSourceCount: { sql: 'source', type: 'countDistinct' }, - + countDistinctApprox: { sql: 'id', type: 'countDistinctApprox' }, - + ratio: { sql: \`\${uniqueSourceCount} / nullif(\${checkinsTotal}, 0)\`, type: 'number' @@ -162,19 +162,19 @@ describe('PreAggregationsAlias', () => { propagateFiltersToSubQuery: true } }, - + segments: { google: { sql: \`source = 'google'\` } }, - - preAggregations: { + + preAggregations: { veryVeryLongTableNameForPreAggregation: { sqlAlias: 'rollupalias', type: 'rollup', - timeDimensionReference: createdAt, - granularity: 'day', + timeDimensionReference: createdAt, + granularity: 'day', measureReferences: [count, revenue], dimensionReferences: [source], }, @@ -186,7 +186,7 @@ describe('PreAggregationsAlias', () => { select * from visitors WHERE \${FILTER_PARAMS.visitors.createdAt.filter('created_at')} \`, sqlAlias: 'rvis', - + joins: { visitor_checkins: { relationship: 'hasMany', @@ -194,7 +194,7 @@ describe('PreAggregationsAlias', () => { } }, - measures: { + measures: { count: { type: 'count' }, @@ -202,22 +202,22 @@ describe('PreAggregationsAlias', () => { sql: 'id', type: 'sum' }, - + checkinsTotal: { sql: \`\${checkinsCount}\`, type: 'sum' }, - + uniqueSourceCount: { sql: 'source', type: 'countDistinct' }, - + countDistinctApprox: { sql: 'id', type: 'countDistinctApprox' }, - + ratio: { sql: \`\${uniqueSourceCount} / nullif(\${checkinsTotal}, 0)\`, type: 'number' @@ -245,20 +245,20 @@ describe('PreAggregationsAlias', () => { propagateFiltersToSubQuery: true } }, - + segments: { google: { sql: \`source = 'google'\` } }, - - preAggregations: { + + preAggregations: { veryVeryLongTableNameForPreAggregation: { sqlAlias: 'rollupalias', type: 'rollup', - timeDimensionReference: createdAt, + timeDimensionReference: createdAt, partitionGranularity: 'month', - granularity: 'day', + granularity: 'day', measureReferences: [count, revenue], dimensionReferences: [source], }, @@ -269,7 +269,7 @@ describe('PreAggregationsAlias', () => { sql: \` select * from visitor_checkins \`, - + sqlAlias: 'vc', measures: { @@ -297,15 +297,15 @@ describe('PreAggregationsAlias', () => { sql: 'created_at' } }, - + preAggregations: { main: { type: 'originalSql', sqlAlias: 'pma', - }, + }, } }) - + cube('GoogleVisitors', { refreshKey: { immutable: true, @@ -314,7 +314,7 @@ describe('PreAggregationsAlias', () => { sql: \`select v.* from \${visitors.sql()} v where v.source = 'google'\`, sqlAlias: 'googlevis', }) - + cube('GoogleVisitorsLongName', { refreshKey: { immutable: true, @@ -323,7 +323,7 @@ describe('PreAggregationsAlias', () => { sql: \`select v.* from \${visitors.sql()} v where v.source = 'google'\`, sqlAlias: 'veryVeryVeryVeryVeryVeryLongSqlAliasForTestItOnPostgresqlDataBase', }) - + `); function replaceTableName(query, preAggregation, suffix) { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-time.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-time.test.ts index 90d987b573e02..dcaab6b8a66f9 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-time.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations-time.test.ts @@ -1,7 +1,7 @@ /* eslint-disable no-restricted-syntax */ import R from 'ramda'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { PostgresDBRunner } from './PostgresDBRunner'; const SCHEMA_VARIANTS = [ @@ -24,7 +24,7 @@ const SCHEMA_VARIANTS = [ sql: 'created_at' }, }, - + preAggregations: { month: { type: 'rollup', @@ -60,7 +60,7 @@ const SCHEMA_VARIANTS = [ sql: 'created_at' }, }, - + preAggregations: { month: { type: 'rollup', @@ -90,7 +90,7 @@ for (const [index, schema] of Object.entries(SCHEMA_VARIANTS)) { await dbRunner.tearDown(); }); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(schema); + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(schema); function replaceTableName(query, preAggregation, suffix) { const [toReplace, params] = query; diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index b3d191d8f5066..6cf04b61fc0d5 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -1,13 +1,13 @@ import { PreAggregationPartitionRangeLoader } from '@cubejs-backend/query-orchestrator'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; import { BigqueryQuery } from '../../../src/adapter/BigqueryQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('PreAggregations', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from visitors WHERE \${FILTER_PARAMS.visitors.createdAt.filter('created_at')} diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts index e8d6d6e4b2aac..d27742bc73707 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation-logic.test.ts @@ -1,12 +1,12 @@ import { UserError } from '../../../src/compiler/UserError'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('SQL Generation', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` const perVisitorRevenueMeasure = { type: 'number', sql: new Function('visitor_revenue', 'visitor_count', 'return visitor_revenue + "/" + visitor_count') @@ -306,7 +306,7 @@ describe('SQL Generation', () => { }); `); - const aliasedCubesCompilers = /** @type Compilers */ prepareCompiler(` + const aliasedCubesCompilers = /** @type Compilers */ prepareJsCompiler(` cube('LeftLongLongLongLongLongLongLongLongLongLongNameCube', { sql: 'SELECT * FROM LEFT_TABLE', sqlAlias: 'left', diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts index c2bc3d7d93b17..70c161f4d7550 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts @@ -3,14 +3,14 @@ import { UserError } from '../../../src/compiler/UserError'; import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; import { BigqueryQuery } from '../../../src/adapter/BigqueryQuery'; import { PrestodbQuery } from '../../../src/adapter/PrestodbQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; import { createJoinedCubesSchema } from '../../unit/utils'; describe('SQL Generation', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` const perVisitorRevenueMeasure = { type: 'number', sql: new Function('visitor_revenue', 'visitor_count', 'return visitor_revenue + "/" + visitor_count') @@ -3401,7 +3401,7 @@ SELECT 1 AS revenue, cast('2024-01-01' AS timestamp) as time UNION ALL // )); it('columns order for the query with the sub-query', async () => { - const joinedSchemaCompilers = prepareCompiler(createJoinedCubesSchema()); + const joinedSchemaCompilers = prepareJsCompiler(createJoinedCubesSchema()); await joinedSchemaCompilers.compiler.compile(); const query = new PostgresQuery({ joinGraph: joinedSchemaCompilers.joinGraph, diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/sub-query-dimensions.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/sub-query-dimensions.test.ts index 1891b9740999e..f022048271fe1 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/sub-query-dimensions.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/sub-query-dimensions.test.ts @@ -1,11 +1,11 @@ import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('Sub Query Dimensions', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`A\`, { sql: \` SELECT 79 AS id, 1 AS foo_id UNION ALL @@ -15,7 +15,7 @@ cube(\`A\`, { SELECT 83 AS id, 5 AS foo_id UNION ALL SELECT 84 AS id, 6 AS foo_id \`, - + measures: { maxFooId: { sql: \`foo_id\`, @@ -88,7 +88,7 @@ cube(\`C\`, { SELECT 793 AS id, 478 AS bar_id, 38.0 AS important_value UNION ALL SELECT 794 AS id, 478 AS bar_id, 43.5 AS important_value \`, - + measures: { importantValue: { sql: \`important_value\`, diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order-2.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order-2.test.ts index 27146aed738a3..20dac065bec76 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order-2.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order-2.test.ts @@ -1,11 +1,11 @@ import { getEnv } from '@cubejs-backend/shared'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('Views Join Order 2', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` view(\`View\`, { description: 'A view', cubes: [ diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order.test.ts index e4e099aaee9a4..ddafd763ddb33 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/views-join-order.test.ts @@ -1,11 +1,11 @@ import { BaseQuery, PostgresQuery } from '../../../src/adapter'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; describe('Views Join Order', () => { jest.setTimeout(200000); - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`fact\`, { sql: \`SELECT 1 as id, 1 as id_product, 10 as quantity\`, dimensions: { diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/views.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/views.test.ts index d4ebbe0ebc0d3..040883aa5d240 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/views.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/views.test.ts @@ -1,6 +1,6 @@ import { PostgresQuery } from '../../../src/adapter/PostgresQuery'; // import { prepareYamlCompiler } from '../../unit/PrepareCompiler'; -import { prepareCompiler } from '../../unit/PrepareCompiler'; +import { prepareJsCompiler } from '../../unit/PrepareCompiler'; import { dbRunner } from './PostgresDBRunner'; // TODO: move into utils @@ -25,7 +25,7 @@ describe('Views in YAML', () => { describe('Views in JS', () => { jest.setTimeout(200000); - const prepareCompilerResult = prepareCompiler(` + const prepareCompilerResult = prepareJsCompiler(` cube(\`orders\`, { sql: \`SELECT 1 as id, 1 as customer_id, '2022-01-01' as timestamp\`, diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts index bd829ce6ff840..a6abb2eae3a56 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts @@ -23,7 +23,7 @@ cubes: - name: time sql: "{CUBE}.timestamp" type: time - `, { yamlExtension: true }); + `); await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -448,7 +448,7 @@ cubes: rollingWindow: trailing: 7 day offset: start - `, { yamlExtension: true }); + `); await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { @@ -493,7 +493,6 @@ cubes: - name: count type: count `, - {}, { compileContext: { authInfo: null, diff --git a/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.js b/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.ts similarity index 54% rename from packages/cubejs-schema-compiler/test/unit/PrepareCompiler.js rename to packages/cubejs-schema-compiler/test/unit/PrepareCompiler.ts index adcc18e6965f5..dc10233deb903 100644 --- a/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.js +++ b/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.ts @@ -1,20 +1,30 @@ import { prepareCompiler as originalPrepareCompiler } from '../../src/compiler/PrepareCompiler'; -export const prepareCompiler = (content, options) => originalPrepareCompiler({ +export type CompileContent = { + content: string; + fileName: string; +}; + +export const prepareJsCompiler = (content, options = {}) => originalPrepareCompiler({ localPath: () => __dirname, dataSchemaFiles: () => Promise.resolve([ { fileName: 'main.js', content: Array.isArray(content) ? content.join('\r\n') : content } ]) }, { adapter: 'postgres', ...options }); -export const prepareYamlCompiler = (content, yamlExtension, options = {}) => originalPrepareCompiler({ +export const prepareYamlCompiler = (content, options = {}) => originalPrepareCompiler({ localPath: () => __dirname, dataSchemaFiles: () => Promise.resolve([ - { fileName: yamlExtension ? 'main.yaml' : 'main.yml', content } + { fileName: 'main.yml', content } ]) }, { adapter: 'postgres', ...options }); -export const prepareCube = (cubeName, cube, options) => { +// export const prepareCompiler = (content: CompileContent | CompileContent[], options) => originalPrepareCompiler({ +// localPath: () => __dirname, +// dataSchemaFiles: () => Promise.resolve(Array.isArray(content) ? content : [content]), +// }, { adapter: 'postgres', ...options }); + +export const prepareCube = (cubeName, cube, options = {}) => { const fileName = `${cubeName}.js`; const content = `cube(${JSON.stringify(cubeName)}, ${JSON.stringify(cube).replace(/"([^"]+)":/g, '$1:')});`; diff --git a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts index b1c2c27ccb0a5..13fc1cc6575f2 100644 --- a/packages/cubejs-schema-compiler/test/unit/base-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/base-query.test.ts @@ -1,6 +1,6 @@ import moment from 'moment-timezone'; import { BaseQuery, PostgresQuery, MssqlQuery, UserError, CubeStoreQuery } from '../../src'; -import { prepareCompiler, prepareYamlCompiler } from './PrepareCompiler'; +import { prepareJsCompiler, prepareYamlCompiler } from './PrepareCompiler'; import { createCubeSchema, createCubeSchemaWithCustomGranularities, @@ -34,7 +34,7 @@ describe('SQL Generation', () => { }); describe('Common - JS - syntax sugar', () => { - const compilers = /** @type Compilers */ prepareCompiler( + const compilers = /** @type Compilers */ prepareJsCompiler( createCubeSchema({ name: 'cards', sqlTable: 'card_tbl' @@ -420,7 +420,7 @@ describe('SQL Generation', () => { }); describe('Custom granularities', () => { - const compilers = /** @type Compilers */ prepareCompiler( + const compilers = /** @type Compilers */ prepareJsCompiler( createCubeSchemaWithCustomGranularities('orders') ); @@ -933,7 +933,7 @@ describe('SQL Generation', () => { }); describe('Base joins', () => { - const compilers = /** @type Compilers */ prepareCompiler([ + const compilers = /** @type Compilers */ prepareJsCompiler([ createCubeSchema({ name: 'cardsA', sqlTable: 'card_tbl', @@ -1001,7 +1001,7 @@ describe('SQL Generation', () => { }); }); describe('Common - JS', () => { - const compilers = /** @type Compilers */ prepareCompiler( + const compilers = /** @type Compilers */ prepareJsCompiler( createCubeSchema({ name: 'cards', refreshKey: ` @@ -1224,7 +1224,7 @@ describe('SQL Generation', () => { }); describe('refreshKey from schema', () => { - const compilers = /** @type Compilers */ prepareCompiler( + const compilers = /** @type Compilers */ prepareJsCompiler( createCubeSchema({ name: 'cards', refreshKey: ` @@ -1430,7 +1430,7 @@ describe('SQL Generation', () => { }); describe('refreshKey only cube (immutable)', () => { - /** @type Compilers */ prepareCompiler( + /** @type Compilers */ prepareJsCompiler( createCubeSchema({ name: 'cards', refreshKey: ` @@ -1454,7 +1454,7 @@ describe('SQL Generation', () => { }); describe('refreshKey only cube (every)', () => { - const compilers = /** @type Compilers */ prepareCompiler( + const compilers = /** @type Compilers */ prepareJsCompiler( createCubeSchema({ name: 'cards', refreshKey: ` @@ -1539,7 +1539,7 @@ describe('SQL Generation', () => { }); it('refreshKey (sql + every) in cube', async () => { - const compilers = /** @type Compilers */ prepareCompiler( + const compilers = /** @type Compilers */ prepareJsCompiler( createCubeSchema({ name: 'cards', refreshKey: ` @@ -1588,7 +1588,7 @@ describe('SQL Generation', () => { }); it('refreshKey (sql + every) in preAggregation', async () => { - const compilers = /** @type Compilers */ prepareCompiler( + const compilers = /** @type Compilers */ prepareJsCompiler( createCubeSchema({ name: 'cards', refreshKey: '', @@ -2331,7 +2331,7 @@ describe('SQL Generation', () => { describe('Class unit tests', () => { it('Test BaseQuery with unaliased cube', async () => { - const set = /** @type Compilers */ prepareCompiler(` + const set = /** @type Compilers */ prepareJsCompiler(` cube('CamelCaseCube', { sql: 'SELECT * FROM TABLE_NAME', measures: { @@ -2378,7 +2378,7 @@ describe('Class unit tests', () => { }); it('Test BaseQuery with aliased cube', async () => { - const set = /** @type Compilers */ prepareCompiler(` + const set = /** @type Compilers */ prepareJsCompiler(` cube('CamelCaseCube', { sql: 'SELECT * FROM TABLE_NAME', sqlAlias: 'T1', @@ -2427,7 +2427,7 @@ describe('Class unit tests', () => { }); it('Test BaseQuery columns order for the query with the sub-query', async () => { - const joinedSchemaCompilers = prepareCompiler(createJoinedCubesSchema()); + const joinedSchemaCompilers = prepareJsCompiler(createJoinedCubesSchema()); await joinedSchemaCompilers.compiler.compile(); await joinedSchemaCompilers.compiler.compile(); const query = new BaseQuery({ diff --git a/packages/cubejs-schema-compiler/test/unit/extensions.test.ts b/packages/cubejs-schema-compiler/test/unit/extensions.test.ts index 8ea8298377250..fac7bb5877688 100644 --- a/packages/cubejs-schema-compiler/test/unit/extensions.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/extensions.test.ts @@ -1,10 +1,10 @@ import { PostgresQuery } from '../../src/adapter/PostgresQuery'; -import { prepareCompiler } from './PrepareCompiler'; +import { prepareJsCompiler } from './PrepareCompiler'; describe('Extensions', () => { const { compiler, joinGraph, cubeEvaluator - } = prepareCompiler(` + } = prepareJsCompiler(` const Funnels = require('Funnels'); import { dynRef } from 'Reflection'; @@ -49,7 +49,7 @@ describe('Extensions', () => { cube(\`FooBar\`, { extends: VisitorsFunnel, - + measures: { conversionsFraction: { sql: dynRef('conversions', (c) => \`\${c} / 100.0\`), diff --git a/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts b/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts index d4af9934cfc63..f329ecb77882d 100644 --- a/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/hierarchies.test.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; -import { prepareCompiler, prepareYamlCompiler } from './PrepareCompiler'; +import { prepareJsCompiler, prepareYamlCompiler } from './PrepareCompiler'; describe('Cube hierarchies', () => { it('base cases', async () => { @@ -164,7 +164,7 @@ describe('Cube hierarchies', () => { path.join(process.cwd(), '/test/unit/fixtures/orders.js'), 'utf8' ); - const { compiler, metaTransformer } = prepareCompiler(modelContent); + const { compiler, metaTransformer } = prepareJsCompiler(modelContent); await compiler.compile(); diff --git a/packages/cubejs-schema-compiler/test/unit/mongobi-query.test.ts b/packages/cubejs-schema-compiler/test/unit/mongobi-query.test.ts index 0d3a52b3d1f11..016aa8f3c4090 100644 --- a/packages/cubejs-schema-compiler/test/unit/mongobi-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/mongobi-query.test.ts @@ -1,8 +1,8 @@ import { MongoBiQuery } from '../../src/adapter/MongoBiQuery'; -import { prepareCompiler } from './PrepareCompiler'; +import { prepareJsCompiler } from './PrepareCompiler'; describe('MongoBiQuery', () => { - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from visitors diff --git a/packages/cubejs-schema-compiler/test/unit/mssql-query.test.ts b/packages/cubejs-schema-compiler/test/unit/mssql-query.test.ts index 6e6a7b16f0f10..b64153ec8b574 100644 --- a/packages/cubejs-schema-compiler/test/unit/mssql-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/mssql-query.test.ts @@ -1,10 +1,10 @@ import { QueryAlias } from '@cubejs-backend/shared'; import { MssqlQuery } from '../../src/adapter/MssqlQuery'; -import { prepareCompiler } from './PrepareCompiler'; +import { prepareJsCompiler } from './PrepareCompiler'; import { createJoinedCubesSchema } from './utils'; describe('MssqlQuery', () => { - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from visitors @@ -29,7 +29,7 @@ describe('MssqlQuery', () => { type: 'number', primaryKey: true, }, - + createdAt: { type: 'time', sql: 'created_at' @@ -44,7 +44,7 @@ describe('MssqlQuery', () => { cube(\`Deals\`, { sql: \`select * from deals\`, - + measures: { amount: { sql: \`amount\`, @@ -60,31 +60,31 @@ describe('MssqlQuery', () => { } } }) - + cube(\`SalesManagers\`, { sql: \`select * from sales_managers\`, - + joins: { Deals: { relationship: \`hasMany\`, sql: \`\${SalesManagers}.id = \${Deals}.sales_manager_id\` } }, - + measures: { averageDealAmount: { sql: \`\${dealsAmount}\`, type: \`avg\` } }, - + dimensions: { id: { sql: \`id\`, type: \`string\`, primaryKey: true }, - + dealsAmount: { sql: \`\${Deals.amount}\`, type: \`number\`, @@ -94,7 +94,7 @@ describe('MssqlQuery', () => { }); `); - const joinedSchemaCompilers = prepareCompiler(createJoinedCubesSchema()); + const joinedSchemaCompilers = prepareJsCompiler(createJoinedCubesSchema()); it('should group by the created_at field on the calculated granularity for unbounded trailing windows', () => compiler.compile().then(() => { diff --git a/packages/cubejs-schema-compiler/test/unit/postgres-query.test.ts b/packages/cubejs-schema-compiler/test/unit/postgres-query.test.ts index 75f4598287169..3ed37badedaec 100644 --- a/packages/cubejs-schema-compiler/test/unit/postgres-query.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/postgres-query.test.ts @@ -1,9 +1,9 @@ /* eslint-disable no-restricted-syntax */ import { PostgresQuery } from '../../src/adapter/PostgresQuery'; -import { prepareCompiler } from './PrepareCompiler'; +import { prepareJsCompiler } from './PrepareCompiler'; describe('PostgresQuery', () => { - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(` + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` cube(\`visitors\`, { sql: \` select * from visitors diff --git a/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts index 641f2aea699d5..450f809208206 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts @@ -1,11 +1,11 @@ -import { prepareCompiler, prepareYamlCompiler } from './PrepareCompiler'; +import { prepareJsCompiler, prepareYamlCompiler } from './PrepareCompiler'; import { createECommerceSchema, createSchemaYaml } from './utils'; import { PostgresQuery, queryClass, QueryFactory } from '../../src'; describe('pre-aggregations', () => { it('rollupJoin scheduledRefresh', async () => { process.env.CUBEJS_SCHEDULED_REFRESH_DEFAULT = 'true'; - const { compiler, cubeEvaluator } = prepareCompiler( + const { compiler, cubeEvaluator } = prepareJsCompiler( ` cube(\`Users\`, { sql: \`SELECT * FROM public.users\`, @@ -93,7 +93,7 @@ describe('pre-aggregations', () => { }); it('query rollupLambda', async () => { - const { compiler, cubeEvaluator, joinGraph } = prepareCompiler( + const { compiler, cubeEvaluator, joinGraph } = prepareJsCompiler( ` cube(\`Users\`, { sql: \`SELECT * FROM public.users\`, diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index aab99d32ccf97..9fdbc18bf78cf 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -1,9 +1,9 @@ -import { prepareCompiler } from './PrepareCompiler'; +import { prepareJsCompiler } from './PrepareCompiler'; import { createCubeSchema, createCubeSchemaWithCustomGranularities, createCubeSchemaWithAccessPolicy } from './utils'; describe('Schema Testing', () => { const schemaCompile = async () => { - const { compiler, cubeEvaluator } = prepareCompiler( + const { compiler, cubeEvaluator } = prepareJsCompiler( createCubeSchema({ name: 'CubeA', preAggregations: ` @@ -169,7 +169,7 @@ describe('Schema Testing', () => { it('invalid schema', async () => { const logger = jest.fn(); - const { compiler } = prepareCompiler( + const { compiler } = prepareJsCompiler( createCubeSchema({ name: 'CubeA', preAggregations: ` @@ -213,7 +213,7 @@ describe('Schema Testing', () => { }); it('visibility modifier', async () => { - const { compiler, metaTransformer } = prepareCompiler([ + const { compiler, metaTransformer } = prepareJsCompiler([ createCubeSchema({ name: 'CubeA', publicly: false @@ -250,7 +250,7 @@ describe('Schema Testing', () => { }); it('dimensions', async () => { - const { compiler, metaTransformer } = prepareCompiler([ + const { compiler, metaTransformer } = prepareJsCompiler([ createCubeSchema({ name: 'CubeA', publicly: false, @@ -269,7 +269,7 @@ describe('Schema Testing', () => { }); it('descriptions', async () => { - const { compiler, metaTransformer } = prepareCompiler([ + const { compiler, metaTransformer } = prepareJsCompiler([ createCubeSchema({ name: 'CubeA', publicly: false, @@ -295,7 +295,7 @@ describe('Schema Testing', () => { }); it('custom granularities in meta', async () => { - const { compiler, metaTransformer } = prepareCompiler([ + const { compiler, metaTransformer } = prepareJsCompiler([ createCubeSchemaWithCustomGranularities('orders') ]); await compiler.compile(); @@ -331,7 +331,7 @@ describe('Schema Testing', () => { }); it('join types', async () => { - const { compiler, cubeEvaluator } = prepareCompiler([ + const { compiler, cubeEvaluator } = prepareJsCompiler([ createCubeSchema({ name: 'CubeA', joins: `{ @@ -369,7 +369,7 @@ describe('Schema Testing', () => { }); it('valid schema with accessPolicy', async () => { - const { compiler } = prepareCompiler([ + const { compiler } = prepareJsCompiler([ createCubeSchemaWithAccessPolicy('ProtectedCube'), ]); await compiler.compile(); diff --git a/packages/cubejs-schema-compiler/test/unit/transpile-speed.test.ts b/packages/cubejs-schema-compiler/test/unit/transpile-speed.test.ts index 998552cc8a26c..fedb11e341dd5 100644 --- a/packages/cubejs-schema-compiler/test/unit/transpile-speed.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/transpile-speed.test.ts @@ -1,4 +1,4 @@ -import { prepareCompiler } from './PrepareCompiler'; +import { prepareJsCompiler } from './PrepareCompiler'; describe('Test Speed', () => { it('100 cube', async () => { @@ -55,7 +55,7 @@ describe('Test Speed', () => { const startTime = +new Date(); for (let i = 0; i < 10; i++) { - const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(cubeString); + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(cubeString); const result = await compiler.compile(); } const endTime = +new Date(); diff --git a/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts b/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts index 30e48a8a2eeb0..ff9c8ea6e3fc4 100644 --- a/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/transpilers.test.ts @@ -1,9 +1,9 @@ -import { prepareCompiler } from './PrepareCompiler'; +import { prepareJsCompiler } from './PrepareCompiler'; describe('Transpilers', () => { it('CubeCheckDuplicatePropTranspiler', async () => { try { - const { compiler } = prepareCompiler(` + const { compiler } = prepareJsCompiler(` cube(\`Test\`, { sql: 'select * from test', dimensions: { @@ -32,7 +32,7 @@ describe('Transpilers', () => { }); it('CubePropContextTranspiler', async () => { - const { compiler } = prepareCompiler(` + const { compiler } = prepareJsCompiler(` let { securityContext } = COMPILE_CONTEXT; cube(\`Test\`, { From 89b08cd18afd0e72781020c7e9b6ac1fc0217a66 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 13:31:33 +0200 Subject: [PATCH 04/22] add tests for inheritance --- .../test/unit/PrepareCompiler.ts | 10 +- .../unit/__snapshots__/schema.test.ts.snap | 1281 +++++++++++++++++ .../test/unit/fixtures/order_users.yml | 17 + .../test/unit/fixtures/orders_big.js | 102 ++ .../test/unit/fixtures/orders_big.yml | 83 ++ .../test/unit/fixtures/orders_ext.js | 72 + .../test/unit/fixtures/orders_ext.yml | 35 + .../test/unit/schema.test.ts | 320 +++- 8 files changed, 1914 insertions(+), 6 deletions(-) create mode 100644 packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/order_users.yml create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.js create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.yml diff --git a/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.ts b/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.ts index dc10233deb903..1807d97e7a133 100644 --- a/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.ts +++ b/packages/cubejs-schema-compiler/test/unit/PrepareCompiler.ts @@ -5,6 +5,11 @@ export type CompileContent = { fileName: string; }; +export const prepareCompiler = (content: CompileContent | CompileContent[], options = {}) => originalPrepareCompiler({ + localPath: () => __dirname, + dataSchemaFiles: () => Promise.resolve(Array.isArray(content) ? content : [content]), +}, { adapter: 'postgres', ...options }); + export const prepareJsCompiler = (content, options = {}) => originalPrepareCompiler({ localPath: () => __dirname, dataSchemaFiles: () => Promise.resolve([ @@ -19,11 +24,6 @@ export const prepareYamlCompiler = (content, options = {}) => originalPrepareCom ]) }, { adapter: 'postgres', ...options }); -// export const prepareCompiler = (content: CompileContent | CompileContent[], options) => originalPrepareCompiler({ -// localPath: () => __dirname, -// dataSchemaFiles: () => Promise.resolve(Array.isArray(content) ? content : [content]), -// }, { adapter: 'postgres', ...options }); - export const prepareCube = (cubeName, cube, options = {}) => { const fileName = `${cubeName}.js`; const content = `cube(${JSON.stringify(cubeName)}, ${JSON.stringify(cube).replace(/"([^"]+)":/g, '$1:')});`; diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap new file mode 100644 index 0000000000000..8d64d65a5908c --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -0,0 +1,1281 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 1`] = ` +Object { + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 2`] = ` +Object { + "city": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 3`] = ` +Object { + "count": Object { + "ownedByCube": true, + "type": "count", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 4`] = ` +Object { + "count": Object { + "ownedByCube": true, + "type": "count", + }, + "count_distinct": Object { + "ownedByCube": true, + "sql": [Function], + "type": "countDistinct", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 5`] = ` +Object { + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 6`] = ` +Object { + "anotherStatus": Object { + "description": "Just another one", + "ownedByCube": true, + "sql": [Function], + }, + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 7`] = ` +Object { + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 8`] = ` +Object { + "ehlo": Object { + "levels": [Function], + "title": "UnderGround", + }, + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 9`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 10`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, + "mainPreAggs": Object { + "dimensionReferences": [Function], + "external": true, + "measureReferences": [Function], + "scheduledRefresh": true, + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 11`] = ` +Array [ + Object { + "role": "*", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludes": Array [ + "status", + ], + "excludesMembers": Array [ + "ordersExt.status", + ], + "includes": "*", + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.id", + "operator": "equals", + "values": [Function], + }, + ], + }, + }, +] +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.js (with additions) 12`] = ` +Array [ + Object { + "role": "*", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludes": Array [ + "status", + ], + "excludesMembers": Array [ + "ordersExt.status", + ], + "includes": "*", + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.id", + "operator": "equals", + "values": [Function], + }, + ], + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludes": Array [ + "min", + "max", + ], + "excludesMembers": Array [ + "ordersExt.min", + "ordersExt.max", + ], + "includes": "*", + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "manager", + "rowLevel": Object { + "filters": Array [ + Object { + "or": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.location", + "operator": "startsWith", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.location", + "operator": "startsWith", + "values": [Function], + }, + ], + }, + ], + }, + }, +] +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 1`] = ` +Object { + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 2`] = ` +Object { + "city": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 3`] = ` +Object { + "count": Object { + "ownedByCube": true, + "sql": [Function], + "type": "count", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 4`] = ` +Object { + "count": Object { + "ownedByCube": true, + "sql": [Function], + "type": "count", + }, + "count_distinct": Object { + "ownedByCube": true, + "sql": [Function], + "type": "countDistinct", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 5`] = ` +Object { + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 6`] = ` +Object { + "anotherStatus": Object { + "description": "Just another one", + "ownedByCube": true, + "sql": [Function], + }, + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 7`] = ` +Object { + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 8`] = ` +Object { + "ehlo": Object { + "levels": [Function], + "title": "UnderGround", + }, + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 9`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 10`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, + "mainPreAggs": Object { + "dimensionReferences": [Function], + "external": true, + "measureReferences": [Function], + "scheduledRefresh": true, + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 11`] = ` +Array [ + Object { + "role": "common", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludesMembers": Array [], + "includes": Array [ + "status", + ], + "includesMembers": Array [ + "ordersExt.status", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.status", + "operator": "equals", + "values": [Function], + }, + Object { + "or": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "equals", + "values": [Function], + }, + ], + }, + ], + }, + }, +] +`; + +exports[`Schema Testing Inheritance CubeB.js correctly extends cubeA.yml (with additions) 12`] = ` +Array [ + Object { + "role": "common", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludesMembers": Array [], + "includes": Array [ + "status", + ], + "includesMembers": Array [ + "ordersExt.status", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.status", + "operator": "equals", + "values": [Function], + }, + Object { + "or": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "equals", + "values": [Function], + }, + ], + }, + ], + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludes": Array [ + "min", + "max", + ], + "excludesMembers": Array [ + "ordersExt.min", + "ordersExt.max", + ], + "includes": "*", + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "manager", + "rowLevel": Object { + "filters": Array [ + Object { + "or": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.location", + "operator": "startsWith", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.location", + "operator": "startsWith", + "values": [Function], + }, + ], + }, + ], + }, + }, +] +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 1`] = ` +Object { + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 2`] = ` +Object { + "city": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 3`] = ` +Object { + "count": Object { + "ownedByCube": true, + "type": "count", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 4`] = ` +Object { + "count": Object { + "ownedByCube": true, + "type": "count", + }, + "count_distinct": Object { + "ownedByCube": true, + "sql": [Function], + "type": "countDistinct", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 5`] = ` +Object { + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 6`] = ` +Object { + "anotherStatus": Object { + "description": "Just another one", + "ownedByCube": true, + "sql": [Function], + }, + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 7`] = ` +Object { + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 8`] = ` +Object { + "ehlo": Object { + "levels": [Function], + "title": "UnderGround", + }, + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 9`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 10`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, + "mainPreAggs": Object { + "dimensionReferences": [Function], + "external": true, + "measureReferences": [Function], + "scheduledRefresh": true, + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 11`] = ` +Array [ + Object { + "role": "*", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludes": Array [ + "status", + ], + "excludesMembers": Array [ + "ordersExt.status", + ], + "includes": "*", + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.id", + "operator": "equals", + "values": [Function], + }, + ], + }, + }, +] +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.js (with additions) 12`] = ` +Array [ + Object { + "role": "*", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludes": Array [ + "status", + ], + "excludesMembers": Array [ + "ordersExt.status", + ], + "includes": "*", + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.id", + "operator": "equals", + "values": [Function], + }, + ], + }, + }, + Object { + "memberLevel": Object { + "excludes": Array [ + "status", + ], + "excludesMembers": Array [ + "ordersExt.status", + ], + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "manager", + }, +] +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 1`] = ` +Object { + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 2`] = ` +Object { + "city": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "completed_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "created_at": Object { + "ownedByCube": true, + "sql": [Function], + "type": "time", + }, + "id": Object { + "ownedByCube": true, + "primaryKey": true, + "sql": [Function], + "type": "number", + }, + "status": Object { + "ownedByCube": true, + "sql": [Function], + "type": "string", + }, + "user_id": Object { + "ownedByCube": true, + "sql": [Function], + "type": "number", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 3`] = ` +Object { + "count": Object { + "ownedByCube": true, + "sql": [Function], + "type": "count", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 4`] = ` +Object { + "count": Object { + "ownedByCube": true, + "sql": [Function], + "type": "count", + }, + "count_distinct": Object { + "ownedByCube": true, + "sql": [Function], + "type": "countDistinct", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 5`] = ` +Object { + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 6`] = ` +Object { + "anotherStatus": Object { + "description": "Just another one", + "ownedByCube": true, + "sql": [Function], + }, + "sfUsers": Object { + "description": "SF users segment from createCubeSchema", + "ownedByCube": true, + "sql": [Function], + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 7`] = ` +Object { + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 8`] = ` +Object { + "ehlo": Object { + "levels": [Function], + "title": "UnderGround", + }, + "hello": Object { + "levels": [Function], + "title": "World", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 9`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 10`] = ` +Object { + "countCreatedAt": Object { + "external": true, + "granularity": "day", + "measureReferences": [Function], + "partitionGranularity": "month", + "refreshKey": Object { + "every": "1 hour", + }, + "scheduledRefresh": true, + "timeDimensionReference": [Function], + "type": "rollup", + }, + "mainPreAggs": Object { + "dimensionReferences": [Function], + "external": true, + "measureReferences": [Function], + "scheduledRefresh": true, + "type": "rollup", + }, +} +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 11`] = ` +Array [ + Object { + "role": "common", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludesMembers": Array [], + "includes": Array [ + "status", + ], + "includesMembers": Array [ + "ordersExt.status", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.status", + "operator": "equals", + "values": [Function], + }, + Object { + "or": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "equals", + "values": [Function], + }, + ], + }, + ], + }, + }, +] +`; + +exports[`Schema Testing Inheritance CubeB.yml correctly extends cubeA.yml (with additions) 12`] = ` +Array [ + Object { + "role": "common", + "rowLevel": Object { + "allowAll": true, + }, + }, + Object { + "conditions": Array [ + Object { + "if": [Function], + }, + ], + "memberLevel": Object { + "excludesMembers": Array [], + "includes": Array [ + "status", + ], + "includesMembers": Array [ + "ordersExt.status", + ], + }, + "role": "admin", + "rowLevel": Object { + "filters": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.status", + "operator": "equals", + "values": [Function], + }, + Object { + "or": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.created_at", + "operator": "equals", + "values": [Function], + }, + ], + }, + ], + }, + }, + Object { + "memberLevel": Object { + "excludes": Array [ + "status", + ], + "excludesMembers": Array [ + "ordersExt.status", + ], + "includesMembers": Array [ + "ordersExt.count", + "ordersExt.count_distinct", + "ordersExt.id", + "ordersExt.user_id", + "ordersExt.status", + "ordersExt.created_at", + "ordersExt.completed_at", + "ordersExt.city", + "ordersExt.sfUsers", + "ordersExt.anotherStatus", + ], + }, + "role": "manager", + }, +] +`; diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/order_users.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/order_users.yml new file mode 100644 index 0000000000000..eaf60079d3765 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/order_users.yml @@ -0,0 +1,17 @@ +cubes: + - name: order_users + sql: SELECT * FROM order_users; + + measures: + - name: count + sql: id + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: name + sql: name + type: string diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js new file mode 100644 index 0000000000000..014c1029cd70d --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js @@ -0,0 +1,102 @@ +cube('orders', { + sql_table: 'public.orders', + + dimensions: { + id: { + sql: 'id', + type: 'number', + primary_key: true, + }, + + user_id: { + sql: 'user_id', + type: 'number', + }, + + status: { + sql: 'status', + type: 'string', + }, + + created_at: { + sql: 'created_at', + type: 'time', + }, + + completed_at: { + sql: 'completed_at', + type: 'time', + }, + }, + + measures: { + count: { + type: 'count', + }, + }, + + joins: { + order_users: { + relationship: 'many_to_one', + sql: `${CUBE}.user_id = ${order_users}.id`, + } + }, + + segments: { + sfUsers: { + description: 'SF users segment from createCubeSchema', + sql: `${CUBE}.location = 'San Francisco'` + } + }, + + hierarchies: { + hello: { + title: 'World', + levels: [status], + }, + }, + + preAggregations: { + countCreatedAt: { + type: 'rollup', + measureReferences: [count], + timeDimensionReference: created_at, + granularity: `day`, + partitionGranularity: `month`, + refreshKey: { + every: '1 hour', + }, + scheduledRefresh: true, + }, + }, + + accessPolicy: [ + { + role: "*", + rowLevel: { + allowAll: true + } + }, + { + role: 'admin', + conditions: [ + { + if: `true`, + } + ], + rowLevel: { + filters: [ + { + member: `${CUBE}.id`, + operator: 'equals', + values: [`1`, `2`, `3`] + } + ] + }, + memberLevel: { + includes: `*`, + excludes: [`status`] + }, + } + ] +}); diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml new file mode 100644 index 0000000000000..d86af6f3ee77f --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml @@ -0,0 +1,83 @@ +cubes: + - name: orders + sql_table: orders + + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: user_id + sql: user_id + type: number + + - name: status + sql: status + type: string + + - name: created_at + sql: created_at + type: time + + - name: completed_at + sql: completed_at + type: time + + measures: + - name: count + sql: id + type: count + + joins: + - name: order_users + relationship: many_to_one + sql: "${CUBE}.user_id = ${order_users.id}" + + segments: + - name: sfUsers + description: SF users segment from createCubeSchema + sql: "${CUBE}.location = 'San Francisco'" + + hierarchies: + - name: hello + title: World + levels: [status] + + pre_aggregations: + - name: countCreatedAt + type: rollup + measures: + - CUBE.count + time_dimension: created_at + granularity: day + partition_granularity: month + refresh_key: + every: 1 hour + scheduled_refresh: true + + accessPolicy: + - role: common + rowLevel: + allowAll: true + - role: admin + conditions: + - if: "{ !security_context.isBlocked }" + rowLevel: + filters: + - member: status + operator: equals + values: ["completed"] + - or: + - member: "{CUBE}.created_at" + operator: notInDateRange + values: + - 2022-01-01 + - "{ security_context.currentDate }" + - member: "created_at" + operator: equals + values: + - "{ securityContext.currentDate }" + memberLevel: + includes: + - status diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.js b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.js new file mode 100644 index 0000000000000..a79b3e2d4e2e1 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.js @@ -0,0 +1,72 @@ +cube('ordersExt', { + extends: orders, + + dimensions: { + city: { + sql: 'city', + type: 'string', + }, + }, + + measures: { + count_distinct: { + type: 'count_distinct', + sql: 'status', + }, + }, + + segments: { + anotherStatus: { + description: 'Just another one', + sql: `${CUBE}.status = 'Rock and Roll'` + } + }, + + hierarchies: { + ehlo: { + title: 'UnderGround', + levels: [status, city], + }, + }, + + preAggregations: { + mainPreAggs: { + type: 'rollup', + measures: [count_distinct], + dimensions: [city] + } + }, + + accessPolicy: [ + { + role: 'manager', + conditions: [ + { + if: security_context.userId === 1, + } + ], + rowLevel: { + filters: [ + { + or: [ + { + member: `location`, + operator: 'startsWith', + values: [`San`] + }, + { + member: `location`, + operator: 'startsWith', + values: [`Lon`] + } + ] + } + ] + }, + memberLevel: { + includes: `*`, + excludes: [`min`, `max`] + }, + }, + ] +}); diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.yml new file mode 100644 index 0000000000000..d3e4121f0a2d8 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_ext.yml @@ -0,0 +1,35 @@ +cubes: + - name: ordersExt + extends: orders + + dimensions: + - name: city + sql: city + type: string + + measures: + - name: count_distinct + sql: status + type: count_distinct + + segments: + - name: anotherStatus + description: Just another one + sql: ${CUBE}.status = 'Rock and Roll' + + hierarchies: + - name: ehlo + title: UnderGround + levels: [status, city] + + pre_aggregations: + - name: mainPreAggs + type: rollup + measures: [count_distinct] + dimensions: [city] + + accessPolicy: + - role: manager + memberLevel: + excludes: + - status diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 9fdbc18bf78cf..9b2e6f96d071c 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -1,6 +1,10 @@ -import { prepareJsCompiler } from './PrepareCompiler'; +import fs from 'fs'; +import path from 'path'; +import { prepareCompiler, prepareJsCompiler } from './PrepareCompiler'; import { createCubeSchema, createCubeSchemaWithCustomGranularities, createCubeSchemaWithAccessPolicy } from './utils'; +const CUBE_COMPONENTS = ['dimensions', 'measures', 'segments', 'hierarchies', 'preAggregations', 'accessPolicy']; + describe('Schema Testing', () => { const schemaCompile = async () => { const { compiler, cubeEvaluator } = prepareJsCompiler( @@ -375,4 +379,318 @@ describe('Schema Testing', () => { await compiler.compile(); compiler.throwIfAnyErrors(); }); + + describe('Inheritance', () => { + it('CubeB.js correctly extends cubeA.js (no additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.js'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = 'cube(\'ordersExt\', { extends: orders })'; + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersExt, + fileName: 'orders_ext.js', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toEqual(cubeB[c]); + }); + }); + + it('CubeB.js correctly extends cubeA.js (with additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.js'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_ext.js'), + 'utf8' + ); + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersExt, + fileName: 'orders_ext.js', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toMatchSnapshot(); + expect(cubeB[c]).toMatchSnapshot(); + }); + }); + + it('CubeB.yml correctly extends cubeA.yml (no additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.yml'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = ` + cubes: + - name: ordersExt + extends: orders + `; + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.yml', + }, + { + content: ordersExt, + fileName: 'orders_ext.yml', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toEqual(cubeB[c]); + }); + }); + + it('CubeB.yml correctly extends cubeA.yml (with additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.yml'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_ext.yml'), + 'utf8' + ); + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.yml', + }, + { + content: ordersExt, + fileName: 'orders_ext.yml', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toMatchSnapshot(); + expect(cubeB[c]).toMatchSnapshot(); + }); + }); + + it('CubeB.yml correctly extends cubeA.js (no additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.js'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = ` + cubes: + - name: ordersExt + extends: orders + `; + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersExt, + fileName: 'orders_ext.yml', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toEqual(cubeB[c]); + }); + }); + + it('CubeB.yml correctly extends cubeA.js (with additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.js'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_ext.yml'), + 'utf8' + ); + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersExt, + fileName: 'orders_ext.yml', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toMatchSnapshot(); + expect(cubeB[c]).toMatchSnapshot(); + }); + }); + + it('CubeB.js correctly extends cubeA.yml (no additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.yml'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = 'cube(\'ordersExt\', { extends: orders })'; + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.yml', + }, + { + content: ordersExt, + fileName: 'orders_ext.js', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toEqual(cubeB[c]); + }); + }); + + it('CubeB.js correctly extends cubeA.yml (with additions)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.yml'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_ext.js'), + 'utf8' + ); + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.yml', + }, + { + content: ordersExt, + fileName: 'orders_ext.js', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toMatchSnapshot(); + expect(cubeB[c]).toMatchSnapshot(); + }); + }); + }); }); From 5a5867415539d618b438a8609953515e8762dee2 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 14:30:22 +0200 Subject: [PATCH 05/22] hide sql prop from parent cube if needed --- .../src/compiler/CubeSymbols.ts | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index cced8c1e12037..518b578bedb20 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -12,6 +12,10 @@ import type { ErrorReporter } from './ErrorReporter'; interface CubeDefinition { name: string; extends?: string; + sql?: string | (() => string); + // eslint-disable-next-line camelcase + sql_table?: string | (() => string); + sqlTable?: string | (() => string); measures?: Record; dimensions?: Record; segments?: Record; @@ -203,11 +207,21 @@ export class CubeSymbols { if (cubeDefinition.extends) { const superCube = this.resolveSymbolsCall(cubeDefinition.extends, (name: string) => this.cubeReferenceProxy(name)); - Object.setPrototypeOf( - cubeObject, - // eslint-disable-next-line no-underscore-dangle - superCube.__cubeName ? this.getCubeDefinition(superCube.__cubeName) : superCube - ); + // eslint-disable-next-line no-underscore-dangle + const parentCube = superCube.__cubeName ? this.getCubeDefinition(superCube.__cubeName) : superCube; + Object.setPrototypeOf(cubeObject, parentCube); + + // We have 2 different properties that are mutually exclusive: `sqlTable` & `sql` + // And if in extending cube one of them is defined - we need to hide the other from parent cube definition + if (cubeDefinition.sql_table && parentCube.sql) { + cubeObject.sql = undefined; + } else if (cubeDefinition.sqlTable && parentCube.sql) { + cubeObject.sql = undefined; + } else if (cubeDefinition.sql && parentCube.sql_table) { + cubeObject.sql_table = undefined; + } else if (cubeDefinition.sql && parentCube.sqlTable) { + cubeObject.sqlTable = undefined; + } } return cubeObject; From 9614949e48027ceb018914548cafc03386de136c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 14:30:35 +0200 Subject: [PATCH 06/22] add tests for sql override --- .../test/unit/fixtures/orders_big.js | 2 +- .../test/unit/schema.test.ts | 83 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js index 014c1029cd70d..28f788bccf759 100644 --- a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.js @@ -1,5 +1,5 @@ cube('orders', { - sql_table: 'public.orders', + sql: `SELECT * FROM orders`, dimensions: { id: { diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 9b2e6f96d071c..036bd51363c9a 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -692,5 +692,88 @@ describe('Schema Testing', () => { expect(cubeB[c]).toMatchSnapshot(); }); }); + + it('CubeB.js correctly extends cubeA.yml (with sql override)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.yml'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = 'cube(\'ordersExt\', { extends: orders, sql: "SELECT * FROM other_orders" })'; + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.yml', + }, + { + content: ordersExt, + fileName: 'orders_ext.js', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toEqual(cubeB[c]); + }); + + expect(cubeB.sql).toBeTruthy(); + expect(cubeB.sqlTable).toBeFalsy(); + }); + + it('CubeB.yml correctly extends cubeA.js (with sql_table override)', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.js'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersExt = ` + cubes: + - name: ordersExt + sql_table: orders_override + extends: orders + `; + + const { compiler, cubeEvaluator } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersExt, + fileName: 'orders_ext.yml', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + + const cubeA = cubeEvaluator.cubeFromPath('orders'); + const cubeB = cubeEvaluator.cubeFromPath('ordersExt'); + + CUBE_COMPONENTS.forEach(c => { + expect(cubeA[c]).toEqual(cubeB[c]); + }); + + expect(cubeB.sqlTable).toBeTruthy(); + expect(cubeB.sql).toBeFalsy(); + }); }); }); From 1395db0f0893afb43922aa14b4c7f0191e681816 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 15:45:26 +0200 Subject: [PATCH 07/22] fix pre-aggs inheritance --- .../cubejs-schema-compiler/src/compiler/CubeSymbols.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 518b578bedb20..23905cec7ba27 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -128,8 +128,16 @@ export class CubeSymbols { }, get preAggregations() { + // For preAggregations order is important, and destructing parents cube pre-aggs first will lead to + // unexpected results, so we can not use common approach with allDefinitions('preAggregations') here. if (!preAggregations) { - preAggregations = this.allDefinitions('preAggregations'); + const parentPreAggregations = cubeDefinition.extends ? super.preAggregations : null; + + if (parentPreAggregations) { + preAggregations = { ...cubeDefinition.preAggregations, ...parentPreAggregations, ...cubeDefinition.preAggregations }; + } else { + preAggregations = { ...cubeDefinition.preAggregations }; + } } return preAggregations; }, From 0317a4227bf4cfa78d4316464b6c65602c14785d Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 15:45:38 +0200 Subject: [PATCH 08/22] fix test --- .../test/integration/postgres/pre-aggregations.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 6cf04b61fc0d5..8ae367a63688b 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -1199,7 +1199,8 @@ describe('PreAggregations', () => { const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); console.log(JSON.stringify(preAggregationsDescription, null, 2)); - expect(preAggregationsDescription[0].tableName).toEqual('visitors_default'); + // For extended cubes pre-aggregations from parents are treated as local + expect(preAggregationsDescription[0].tableName).toEqual('reference_original_sql_default'); return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { expect(res).toEqual( From f63950086e174a03d6ec39b96c00e6d3bdc3d7bf Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 15:51:32 +0200 Subject: [PATCH 09/22] remove unneeded --- packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 23905cec7ba27..3cd7e174fbc20 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -221,12 +221,8 @@ export class CubeSymbols { // We have 2 different properties that are mutually exclusive: `sqlTable` & `sql` // And if in extending cube one of them is defined - we need to hide the other from parent cube definition - if (cubeDefinition.sql_table && parentCube.sql) { + if (cubeDefinition.sqlTable && parentCube.sql) { cubeObject.sql = undefined; - } else if (cubeDefinition.sqlTable && parentCube.sql) { - cubeObject.sql = undefined; - } else if (cubeDefinition.sql && parentCube.sql_table) { - cubeObject.sql_table = undefined; } else if (cubeDefinition.sql && parentCube.sqlTable) { cubeObject.sqlTable = undefined; } From e92c5e67acb2f0bf9fb73ece145cc348ca5f3f4f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 16:59:49 +0200 Subject: [PATCH 10/22] add more tests for views and errors --- .../test/unit/fixtures/orders_dup_members.js | 42 +++++ .../test/unit/schema.test.ts | 173 ++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/orders_dup_members.js diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_dup_members.js b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_dup_members.js new file mode 100644 index 0000000000000..41dcf8f6556cc --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_dup_members.js @@ -0,0 +1,42 @@ +cube('orders', { + sql_table: 'public.orders', + + dimensions: { + id: { + sql: 'id', + type: 'number', + primary_key: true, + }, + + status: { + sql: 'status', + type: 'string', + }, + + created_at: { + sql: 'created_at', + type: 'time', + }, + + completed_at: { + sql: 'completed_at', + type: 'time', + }, + }, + + measures: { + count: { + type: 'count', + }, + status: { + type: 'count', + }, + }, + + hierarchies: { + hello: { + title: 'World', + levels: [status], + }, + }, +}); diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 036bd51363c9a..f1712a52ea86b 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -372,6 +372,26 @@ describe('Schema Testing', () => { }); }); + it('throws an error on duplicate member names', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_dup_members.js'), + 'utf8' + ); + + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + ]); + + try { + await compiler.compile(); + } catch (e: any) { + expect(e.toString()).toMatch(/status defined more than once/); + } + }); + it('valid schema with accessPolicy', async () => { const { compiler } = prepareJsCompiler([ createCubeSchemaWithAccessPolicy('ProtectedCube'), @@ -380,6 +400,159 @@ describe('Schema Testing', () => { compiler.throwIfAnyErrors(); }); + describe('Views', () => { + it('throws errors for incorrect referenced includes members', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders.js'), + 'utf8' + ); + const ordersView = ` + views: + - name: orders_view + cubes: + - join_path: orders + includes: + - id + - status + - nonexistent1 + - nonexistent2.via.path + `; + + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersView, + fileName: 'order_view.yml', + }, + ]); + + try { + await compiler.compile(); + } catch (e: any) { + expect(e.toString()).toMatch(/Paths aren't allowed in cube includes but 'nonexistent2\.via\.path' provided as include member/); + expect(e.toString()).toMatch(/Member 'nonexistent1' is included in 'orders_view' but not defined in any cube/); + expect(e.toString()).toMatch(/Member 'nonexistent2\.via\.path' is included in 'orders_view' but not defined in any cube/); + } + }); + + it('throws errors for incorrect referenced excludes members', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders.js'), + 'utf8' + ); + const ordersView = ` + views: + - name: orders_view + cubes: + - join_path: orders + includes: "*" + excludes: + - id + - status + - nonexistent3 + - nonexistent4 + `; + + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersView, + fileName: 'order_view.yml', + }, + ]); + + try { + await compiler.compile(); + } catch (e: any) { + expect(e.toString()).toMatch(/Member 'nonexistent3' is included in 'orders_view' but not defined in any cube/); + expect(e.toString()).toMatch(/Member 'nonexistent4' is included in 'orders_view' but not defined in any cube/); + } + }); + + it('throws errors for incorrect referenced excludes members with path', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders.js'), + 'utf8' + ); + const ordersView = ` + views: + - name: orders_view + cubes: + - join_path: orders + includes: "*" + excludes: + - id + - status + - nonexistent5.via.path + `; + + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: ordersView, + fileName: 'order_view.yml', + }, + ]); + + try { + await compiler.compile(); + } catch (e: any) { + expect(e.toString()).toMatch(/Paths aren't allowed in cube excludes but 'nonexistent5\.via\.path' provided as exclude member/); + } + }); + + it('throws errors for conflicting members of included cubes', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_big.js'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const ordersView = ` + views: + - name: orders_view + cubes: + - join_path: orders + includes: "*" + - join_path: orders.order_users + includes: "*" + `; + + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + { + content: ordersView, + fileName: 'order_view.yml', + }, + ]); + + try { + await compiler.compile(); + } catch (e: any) { + expect(e.toString()).toMatch(/Included member 'count' conflicts with existing member of 'orders_view'\. Please consider excluding this member or assigning it an alias/); + expect(e.toString()).toMatch(/Included member 'id' conflicts with existing member of 'orders_view'\. Please consider excluding this member or assigning it an alias/); + } + }); + }); + describe('Inheritance', () => { it('CubeB.js correctly extends cubeA.js (no additions)', async () => { const orders = fs.readFileSync( From 82ccf7449dcc07f82d831e43b742a328f2627a2e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 18:01:21 +0200 Subject: [PATCH 11/22] fix preaggs again --- .../src/compiler/CubeSymbols.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts index 3cd7e174fbc20..447564d81af73 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeSymbols.ts @@ -21,6 +21,8 @@ interface CubeDefinition { segments?: Record; hierarchies?: Record; preAggregations?: Record; + // eslint-disable-next-line camelcase + pre_aggregations?: Record; joins?: Record; accessPolicy?: any[]; includes?: any; @@ -132,11 +134,13 @@ export class CubeSymbols { // unexpected results, so we can not use common approach with allDefinitions('preAggregations') here. if (!preAggregations) { const parentPreAggregations = cubeDefinition.extends ? super.preAggregations : null; + // Unfortunately, cube is not camelized yet at this point :( + const localPreAggregations = cubeDefinition.preAggregations || cubeDefinition.pre_aggregations; if (parentPreAggregations) { - preAggregations = { ...cubeDefinition.preAggregations, ...parentPreAggregations, ...cubeDefinition.preAggregations }; + preAggregations = { ...localPreAggregations, ...parentPreAggregations, ...localPreAggregations }; } else { - preAggregations = { ...cubeDefinition.preAggregations }; + preAggregations = { ...localPreAggregations }; } } return preAggregations; @@ -221,9 +225,10 @@ export class CubeSymbols { // We have 2 different properties that are mutually exclusive: `sqlTable` & `sql` // And if in extending cube one of them is defined - we need to hide the other from parent cube definition - if (cubeDefinition.sqlTable && parentCube.sql) { + // Unfortunately, cube is not camelized yet at this point :( + if ((cubeDefinition.sqlTable || cubeDefinition.sql_table) && parentCube.sql) { cubeObject.sql = undefined; - } else if (cubeDefinition.sql && parentCube.sqlTable) { + } else if (cubeDefinition.sql && (parentCube.sqlTable || parentCube.sql_table)) { cubeObject.sqlTable = undefined; } } From 61d0501a139b2e7f00e86a511c143a503abd73d4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 18:26:39 +0200 Subject: [PATCH 12/22] mooooore tests :) --- .../unit/fixtures/orders_incorrect_acl.yml | 83 +++++++++++++++++++ .../unit/fixtures/orders_nonexist_acl.yml | 83 +++++++++++++++++++ .../test/unit/schema.test.ts | 81 +++++++++++++++--- 3 files changed, 237 insertions(+), 10 deletions(-) create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/orders_incorrect_acl.yml create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/orders_nonexist_acl.yml diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_incorrect_acl.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_incorrect_acl.yml new file mode 100644 index 0000000000000..ddb349d3bdddd --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_incorrect_acl.yml @@ -0,0 +1,83 @@ +cubes: + - name: orders + sql_table: orders + + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: user_id + sql: user_id + type: number + + - name: status + sql: status + type: string + + - name: created_at + sql: created_at + type: time + + - name: completed_at + sql: completed_at + type: time + + measures: + - name: count + sql: id + type: count + + joins: + - name: order_users + relationship: many_to_one + sql: "${CUBE}.user_id = ${order_users.id}" + + segments: + - name: sfUsers + description: SF users segment from createCubeSchema + sql: "${CUBE}.location = 'San Francisco'" + + hierarchies: + - name: hello + title: World + levels: [status] + + pre_aggregations: + - name: countCreatedAt + type: rollup + measures: + - CUBE.count + time_dimension: created_at + granularity: day + partition_granularity: month + refresh_key: + every: 1 hour + scheduled_refresh: true + + accessPolicy: + - role: common + rowLevel: + allowAll: true + - role: admin + conditions: + - if: "{ !security_context.isBlocked }" + rowLevel: + filters: + - member: "{CUBE}.order_users.name" + operator: equals + values: ["completed"] + - or: + - member: "{CUBE}.created_at" + operator: notInDateRange + values: + - 2022-01-01 + - "{ security_context.currentDate }" + - member: "created_at" + operator: equals + values: + - "{ securityContext.currentDate }" + memberLevel: + includes: + - status diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_nonexist_acl.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_nonexist_acl.yml new file mode 100644 index 0000000000000..762496140b488 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_nonexist_acl.yml @@ -0,0 +1,83 @@ +cubes: + - name: orders + sql_table: orders + + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: user_id + sql: user_id + type: number + + - name: status + sql: status + type: string + + - name: created_at + sql: created_at + type: time + + - name: completed_at + sql: completed_at + type: time + + measures: + - name: count + sql: id + type: count + + joins: + - name: order_users + relationship: many_to_one + sql: "${CUBE}.user_id = ${order_users.id}" + + segments: + - name: sfUsers + description: SF users segment from createCubeSchema + sql: "${CUBE}.location = 'San Francisco'" + + hierarchies: + - name: hello + title: World + levels: [status] + + pre_aggregations: + - name: countCreatedAt + type: rollup + measures: + - CUBE.count + time_dimension: created_at + granularity: day + partition_granularity: month + refresh_key: + every: 1 hour + scheduled_refresh: true + + accessPolicy: + - role: common + rowLevel: + allowAll: true + - role: admin + conditions: + - if: "{ !security_context.isBlocked }" + rowLevel: + filters: + - member: "{CUBE}.other.path.created_at" + operator: equals + values: ["completed"] + - or: + - member: "{CUBE}.created_at" + operator: notInDateRange + values: + - 2022-01-01 + - "{ security_context.currentDate }" + - member: "created_at" + operator: equals + values: + - "{ securityContext.currentDate }" + memberLevel: + includes: + - status diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index f1712a52ea86b..25b1155f9a579 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -392,12 +392,70 @@ describe('Schema Testing', () => { } }); - it('valid schema with accessPolicy', async () => { - const { compiler } = prepareJsCompiler([ - createCubeSchemaWithAccessPolicy('ProtectedCube'), - ]); - await compiler.compile(); - compiler.throwIfAnyErrors(); + describe('Access Policies', () => { + it('valid schema with accessPolicy', async () => { + const { compiler } = prepareJsCompiler([ + createCubeSchemaWithAccessPolicy('ProtectedCube'), + ]); + await compiler.compile(); + compiler.throwIfAnyErrors(); + }); + + it('throw errors for nonexistent policy members with paths', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_nonexist_acl.yml'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.yml', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + + try { + await compiler.compile(); + throw new Error('should throw earlier'); + } catch (e: any) { + expect(e.toString()).toMatch(/orders.other cannot be resolved. There's no such member or cube/); + } + }); + + it('throw errors for incorrect policy members with paths', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_incorrect_acl.yml'), + 'utf8' + ); + const orderUsers = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/order_users.yml'), + 'utf8' + ); + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.yml', + }, + { + content: orderUsers, + fileName: 'order_users.yml', + }, + ]); + + try { + await compiler.compile(); + throw new Error('should throw earlier'); + } catch (e: any) { + expect(e.toString()).toMatch(/Paths aren't allowed in the accessPolicy policy but 'order_users.name' provided as a filter member reference for orders/); + } + }); }); describe('Views', () => { @@ -431,6 +489,7 @@ describe('Schema Testing', () => { try { await compiler.compile(); + throw new Error('should throw earlier'); } catch (e: any) { expect(e.toString()).toMatch(/Paths aren't allowed in cube includes but 'nonexistent2\.via\.path' provided as include member/); expect(e.toString()).toMatch(/Member 'nonexistent1' is included in 'orders_view' but not defined in any cube/); @@ -438,7 +497,7 @@ describe('Schema Testing', () => { } }); - it('throws errors for incorrect referenced excludes members', async () => { + it('throws errors for incorrect referenced excludes members with paths', async () => { const orders = fs.readFileSync( path.join(process.cwd(), '/test/unit/fixtures/orders.js'), 'utf8' @@ -452,7 +511,7 @@ describe('Schema Testing', () => { excludes: - id - status - - nonexistent3 + - nonexistent3.ext - nonexistent4 `; @@ -469,9 +528,9 @@ describe('Schema Testing', () => { try { await compiler.compile(); + throw new Error('should throw earlier'); } catch (e: any) { - expect(e.toString()).toMatch(/Member 'nonexistent3' is included in 'orders_view' but not defined in any cube/); - expect(e.toString()).toMatch(/Member 'nonexistent4' is included in 'orders_view' but not defined in any cube/); + expect(e.toString()).toMatch(/Paths aren't allowed in cube excludes but 'nonexistent3.ext' provided as exclude member/); } }); @@ -505,6 +564,7 @@ describe('Schema Testing', () => { try { await compiler.compile(); + throw new Error('should throw earlier'); } catch (e: any) { expect(e.toString()).toMatch(/Paths aren't allowed in cube excludes but 'nonexistent5\.via\.path' provided as exclude member/); } @@ -546,6 +606,7 @@ describe('Schema Testing', () => { try { await compiler.compile(); + throw new Error('should throw earlier'); } catch (e: any) { expect(e.toString()).toMatch(/Included member 'count' conflicts with existing member of 'orders_view'\. Please consider excluding this member or assigning it an alias/); expect(e.toString()).toMatch(/Included member 'id' conflicts with existing member of 'orders_view'\. Please consider excluding this member or assigning it an alias/); From af796de591127b343e3109c58ec75d3d378be176 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 18:29:43 +0200 Subject: [PATCH 13/22] more test coverage --- .../unit/__snapshots__/schema.test.ts.snap | 64 +++++++++++++++++++ .../test/unit/fixtures/orders_big.yml | 10 +++ 2 files changed, 74 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap index 8d64d65a5908c..3681c0aa79b51 100644 --- a/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap +++ b/packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap @@ -557,6 +557,22 @@ Array [ }, ], }, + Object { + "and": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "equals", + "values": [Function], + }, + ], + }, ], }, }, @@ -611,6 +627,22 @@ Array [ }, ], }, + Object { + "and": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "equals", + "values": [Function], + }, + ], + }, ], }, }, @@ -1197,6 +1229,22 @@ Array [ }, ], }, + Object { + "and": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "equals", + "values": [Function], + }, + ], + }, ], }, }, @@ -1251,6 +1299,22 @@ Array [ }, ], }, + Object { + "and": Array [ + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "notInDateRange", + "values": [Function], + }, + Object { + "member": [Function], + "memberReference": "ordersExt.completed_at", + "operator": "equals", + "values": [Function], + }, + ], + }, ], }, }, diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml index d86af6f3ee77f..5e96c1f8b765c 100644 --- a/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/orders_big.yml @@ -78,6 +78,16 @@ cubes: operator: equals values: - "{ securityContext.currentDate }" + - and: + - member: "{CUBE}.completed_at" + operator: notInDateRange + values: + - 2022-01-01 + - "{ security_context.currentDate }" + - member: "completed_at" + operator: equals + values: + - "{ securityContext.currentDate }" memberLevel: includes: - status From 4c26d5b0a839c84e503a58480e61f6330f376c0d Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 18:56:36 +0200 Subject: [PATCH 14/22] more tests for folders --- .../unit/fixtures/folders_invalid_path.yml | 102 ++++++++++++++++++ .../test/unit/fixtures/folders_non_exist.yml | 102 ++++++++++++++++++ .../test/unit/folders.test.ts | 17 +++ 3 files changed, 221 insertions(+) create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/folders_invalid_path.yml create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/folders_non_exist.yml diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/folders_invalid_path.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/folders_invalid_path.yml new file mode 100644 index 0000000000000..b9df376e84aa5 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/folders_invalid_path.yml @@ -0,0 +1,102 @@ +cubes: + - name: orders + sql: SELECT * FROM orders + joins: + - name: users + sql: "{CUBE}.order_id = {orders}.id" + relationship: many_to_one + measures: + - name: count + sql: id + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: number + sql: number + type: number + + - name: status + sql: status + type: string + hierarchies: + - name: orders_hierarchy + levels: + - "{CUBE}.status" + - number + - users.city + - name: some_other_hierarchy + title: Some other hierarchy + levels: + - users.state + - users.city + # + - name: users + sql: SELECT * FROM users + hierarchies: + - name: users_hierarchy + levels: + - users.age + - city + dimensions: + - name: age + sql: age + type: number + - name: state + sql: state + type: string + - name: city + sql: city + type: string + - name: gender + sql: gender + type: string + +views: + - name: test_view + cubes: + - join_path: orders + includes: "*" + - join_path: users + includes: + - age + - state + - name: gender + alias: renamed_gender + folders: + - name: folder1 + includes: + - age + - renamed_gender + - users.age + - name: folder2 + includes: '*' + - name: test_view2 + cubes: + - join_path: orders + alias: renamed_orders + prefix: true + includes: "*" + - join_path: users + prefix: true + includes: + - age + - state + folders: + - name: folder1 + includes: + - users_age + - users_state + - renamed_orders_status + # - name: empty_view + # cubes: + # - join_path: orders + # includes: + # - count + # - status + + + diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/folders_non_exist.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/folders_non_exist.yml new file mode 100644 index 0000000000000..6915be39b1b61 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/folders_non_exist.yml @@ -0,0 +1,102 @@ +cubes: + - name: orders + sql: SELECT * FROM orders + joins: + - name: users + sql: "{CUBE}.order_id = {orders}.id" + relationship: many_to_one + measures: + - name: count + sql: id + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: number + sql: number + type: number + + - name: status + sql: status + type: string + hierarchies: + - name: orders_hierarchy + levels: + - "{CUBE}.status" + - number + - users.city + - name: some_other_hierarchy + title: Some other hierarchy + levels: + - users.state + - users.city + # + - name: users + sql: SELECT * FROM users + hierarchies: + - name: users_hierarchy + levels: + - users.age + - city + dimensions: + - name: age + sql: age + type: number + - name: state + sql: state + type: string + - name: city + sql: city + type: string + - name: gender + sql: gender + type: string + +views: + - name: test_view + cubes: + - join_path: orders + includes: "*" + - join_path: users + includes: + - age + - state + - name: gender + alias: renamed_gender + folders: + - name: folder1 + includes: + - age + - renamed_gender + - non-existent + - name: folder2 + includes: '*' + - name: test_view2 + cubes: + - join_path: orders + alias: renamed_orders + prefix: true + includes: "*" + - join_path: users + prefix: true + includes: + - age + - state + folders: + - name: folder1 + includes: + - users_age + - users_state + - renamed_orders_status + # - name: empty_view + # cubes: + # - join_path: orders + # includes: + # - count + # - status + + + diff --git a/packages/cubejs-schema-compiler/test/unit/folders.test.ts b/packages/cubejs-schema-compiler/test/unit/folders.test.ts index 00e93426e83f6..cb9342cd138c2 100644 --- a/packages/cubejs-schema-compiler/test/unit/folders.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/folders.test.ts @@ -43,6 +43,23 @@ describe('Cube Folders', () => { ); }); + it('throws errors for folder members with path', async () => { + const modelContent = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/folders_invalid_path.yml'), + 'utf8' + ); + // eslint-disable-next-line @typescript-eslint/no-shadow + const { compiler } = prepareYamlCompiler(modelContent); + + try { + await compiler.compile(); + throw new Error('should throw earlier'); + } catch (e: any) { + expect(e.toString()).toMatch(/Paths aren't allowed in the 'folders' but 'users.age' has been provided for test_view/); + expect(e.toString()).toMatch(/Member 'users.age' included in folder 'folder1' not found/); + } + }); + it('a folder with aliased and prefixed cubes', async () => { const view = metaTransformer.cubes.find( (it) => it.config.name === 'test_view2' From 97d9573dd2194a8368d6abd0cd58382535bd156f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 25 Mar 2025 19:15:07 +0200 Subject: [PATCH 15/22] remove useless check --- .../src/compiler/CubeEvaluator.ts | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index c58823b4ac9c6..cdd27b2800a53 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -221,25 +221,17 @@ export class CubeEvaluator extends CubeSymbols { } private prepareHierarchies(cube: any, errorReporter: ErrorReporter): void { - const uniqueHierarchyNames = new Set(); if (Object.keys(cube.hierarchies).length) { - cube.evaluatedHierarchies = Object.entries(cube.hierarchies).map(([name, hierarchy]) => { - if (uniqueHierarchyNames.has(name)) { - errorReporter.error(`Duplicate hierarchy name '${name}' in cube '${cube.name}'`); - } - uniqueHierarchyNames.add(name); - - return ({ - name, - ...(typeof hierarchy === 'object' ? hierarchy : {}), - levels: this.evaluateReferences( - cube.name, - // @ts-ignore - hierarchy.levels, - { originalSorting: true } - ) - }); - }); + cube.evaluatedHierarchies = Object.entries(cube.hierarchies).map(([name, hierarchy]) => ({ + name, + ...(typeof hierarchy === 'object' ? hierarchy : {}), + levels: this.evaluateReferences( + cube.name, + // @ts-ignore + hierarchy.levels, + { originalSorting: true } + ) + })); } if (cube.isView && (cube.includedMembers || []).length) { From ad11e367dd633cc470e5e83c0f9e84d8e293fc67 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 27 Mar 2025 23:04:27 +0200 Subject: [PATCH 16/22] just refactoring --- .../src/compiler/CubeValidator.ts | 151 +++++++++--------- 1 file changed, 78 insertions(+), 73 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index 126247c5f42af..a356d957f8164 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -42,6 +42,30 @@ function formatStatePath(state: Joi.State): string { return ''; } +function condition(fun, then, otherwise) { + return Joi.alternatives().conditional( + Joi.ref('.'), { + is: Joi.custom((value, helper) => (fun(value) ? value : helper.message({}))), + then, + otherwise + } + ); +} + +function defined(a) { + return typeof a !== 'undefined'; +} + +function inherit(a, b) { + return Joi.object().keys({ ...a, ...b }); +} + +function requireOneOf(...keys) { + return Joi.alternatives().try( + ...(keys.map((k) => Joi.object().keys({ [k]: Joi.exist().required() }))) + ); +} + const regexTimeInterval = Joi.string().custom((value, helper) => { if (value.match(/^(-?\d+) (minute|hour|day|week|month|quarter|year)s?$/)) { return value; @@ -165,10 +189,11 @@ const BaseDimensionWithoutSubQuery = { }) }; -const BaseDimension = Object.assign({ +const BaseDimension = { subQuery: Joi.boolean().strict(), - propagateFiltersToSubQuery: Joi.boolean().strict() -}, BaseDimensionWithoutSubQuery); + propagateFiltersToSubQuery: Joi.boolean().strict(), + ...BaseDimensionWithoutSubQuery +}; const FixedRollingWindow = { type: Joi.string().valid('fixed'), @@ -234,30 +259,6 @@ const BaseMeasure = { meta: Joi.any() }; -function condition(fun, then, otherwise) { - return Joi.alternatives().conditional( - Joi.ref('.'), { - is: Joi.custom((value, helper) => (fun(value) ? value : helper.message({}))), - then, - otherwise - } - ); -} - -function defined(a) { - return typeof a !== 'undefined'; -} - -function inherit(a, b) { - return Joi.object().keys(Object.assign({}, a, b)); -} - -function requireOneOf(...keys) { - return Joi.alternatives().try( - ...(keys.map((k) => Joi.object().keys({ [k]: Joi.exist().required() }))) - ); -} - const PreAggregationRefreshKeySchema = condition( (s) => defined(s.sql), Joi.object().keys({ @@ -605,6 +606,47 @@ const MeasuresSchema = Joi.object().pattern(identifierRegex, Joi.alternatives(). ] )); +const DimensionsSchema = Joi.object().pattern(identifierRegex, Joi.alternatives().try( + inherit(BaseDimensionWithoutSubQuery, { + case: Joi.object().keys({ + when: Joi.array().items(Joi.object().keys({ + sql: Joi.func().required(), + label: Joi.alternatives([ + Joi.string(), + Joi.object().keys({ + sql: Joi.func().required() + }) + ]) + })), + else: Joi.object().keys({ + label: Joi.alternatives([ + Joi.string(), + Joi.object().keys({ + sql: Joi.func().required() + }) + ]) + }) + }).required() + }), + inherit(BaseDimensionWithoutSubQuery, { + latitude: Joi.object().keys({ + sql: Joi.func().required() + }).required(), + longitude: Joi.object().keys({ + sql: Joi.func().required() + }).required() + }), + inherit(BaseDimension, { + sql: Joi.func().required(), + }), + inherit(BaseDimension, { + multiStage: Joi.boolean().valid(true), + type: Joi.any().valid('number').required(), + sql: Joi.func().required(), + addGroupBy: Joi.func(), + }) +)); + const SegmentsSchema = Joi.object().pattern(identifierRegex, Joi.object().keys({ aliases: Joi.array().items(Joi.string()), sql: Joi.func().required(), @@ -713,46 +755,7 @@ const baseSchema = { ).required() })), measures: MeasuresSchema, - dimensions: Joi.object().pattern(identifierRegex, Joi.alternatives().try( - inherit(BaseDimensionWithoutSubQuery, { - case: Joi.object().keys({ - when: Joi.array().items(Joi.object().keys({ - sql: Joi.func().required(), - label: Joi.alternatives([ - Joi.string(), - Joi.object().keys({ - sql: Joi.func().required() - }) - ]) - })), - else: Joi.object().keys({ - label: Joi.alternatives([ - Joi.string(), - Joi.object().keys({ - sql: Joi.func().required() - }) - ]) - }) - }).required() - }), - inherit(BaseDimensionWithoutSubQuery, { - latitude: Joi.object().keys({ - sql: Joi.func().required() - }).required(), - longitude: Joi.object().keys({ - sql: Joi.func().required() - }).required() - }), - inherit(BaseDimension, { - sql: Joi.func().required() - }), - inherit(BaseDimension, { - multiStage: Joi.boolean().valid(true), - type: Joi.any().valid('number').required(), - sql: Joi.func().required(), - addGroupBy: Joi.func(), - }) - )), + dimensions: DimensionsSchema, segments: SegmentsSchema, preAggregations: PreAggregationsAlternatives, folders: Joi.array().items(Joi.object().keys({ @@ -765,14 +768,16 @@ const baseSchema = { accessPolicy: Joi.array().items(RolePolicySchema.required()), }; +const hierarchySchema = Joi.object().pattern(identifierRegex, Joi.object().keys({ + title: Joi.string(), + public: Joi.boolean().strict(), + levels: Joi.func() +})); + const cubeSchema = inherit(baseSchema, { sql: Joi.func(), sqlTable: Joi.func(), - hierarchies: Joi.object().pattern(identifierRegex, Joi.object().keys({ - title: Joi.string(), - public: Joi.boolean().strict(), - levels: Joi.func() - })) + hierarchies: hierarchySchema, }).xor('sql', 'sqlTable').messages({ 'object.xor': 'You must use either sql or sqlTable within a model, but not both' }); @@ -803,7 +808,7 @@ const viewSchema = inherit(baseSchema, { }) ), accessPolicy: Joi.array().items(RolePolicySchema.required()), - hierarchies: Joi.any() + hierarchies: hierarchySchema, }); function formatErrorMessageFromDetails(explain, d) { From 0c83b0e38c941856e71baca5a3c1cd2dc28d7341 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 27 Mar 2025 23:05:01 +0200 Subject: [PATCH 17/22] fix schema validation for extended cubes --- .../cubejs-schema-compiler/src/compiler/CubeValidator.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts index a356d957f8164..4010edf853aba 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts @@ -885,7 +885,11 @@ export class CubeValidator { } public validate(cube, errorReporter: ErrorReporter) { - const result = cube.isView ? viewSchema.validate(cube) : cubeSchema.validate(cube); + const options = { + nonEnumerables: true, + abortEarly: false, // This will allow all errors to be reported, not just the first one + }; + const result = cube.isView ? viewSchema.validate(cube, options) : cubeSchema.validate(cube, options); if (result.error != null) { errorReporter.error(formatErrorMessage(result.error), result.error); From 16c24ecc37e32a5e67d7b59a818e4f3745bb5ecc Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 27 Mar 2025 23:05:10 +0200 Subject: [PATCH 18/22] add tests --- .../test/unit/fixtures/invalid_cubes.yaml | 45 +++++++++++++++++++ .../test/unit/schema.test.ts | 27 +++++++++++ 2 files changed, 72 insertions(+) create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/invalid_cubes.yaml diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/invalid_cubes.yaml b/packages/cubejs-schema-compiler/test/unit/fixtures/invalid_cubes.yaml new file mode 100644 index 0000000000000..f4ce3223a76dd --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/invalid_cubes.yaml @@ -0,0 +1,45 @@ +cubes: + - name: base_cube + sql_table: foo + + dimensions: + - name: status + sql: status + type: string + + - name: parent_dim_no_type + sql: barbus + + - name: parent_dim_bad_type + type: bad_type + sql: badus + + - name: parent_dim_no_sql + type: string + + measures: + - name: count + type: count + + - name: parent_meas_no_type + + - name: parent_meas_bad_type + type: bad_type + + - name: child_cube + extends: base_cube + + dimensions: + - name: bar + sql: bar + type: string + + - name: child_dim_no_type + sql: barbus + + - name: child_dim_bad_type + type: bad_type + sql: shmarbus + + - name: child_dim_no_sql + type: number diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 25b1155f9a579..025f3ad2ea5df 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -1009,5 +1009,32 @@ describe('Schema Testing', () => { expect(cubeB.sqlTable).toBeTruthy(); expect(cubeB.sql).toBeFalsy(); }); + + it('throws error for member without type in cubeB extending cubeA', async () => { + const cubes = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/invalid_cubes.yaml'), + 'utf8' + ); + const { compiler } = prepareCompiler([ + { + content: cubes, + fileName: 'invalid_cubes.yaml', + }, + ]); + + try { + await compiler.compile(); + throw new Error('should throw earlier'); + } catch (e: any) { + expect(e.toString()).toMatch(/"measures\.parent_meas_no_type\.sql" is required/); + expect(e.toString()).toMatch(/"measures\.parent_meas_no_type\.type" is required/); + expect(e.toString()).toMatch(/"measures\.parent_meas_bad_type\.type" must be one of/); + expect(e.toString()).toMatch(/"dimensions\.parent_dim_no_type" does not match any of the allowed types/); + expect(e.toString()).toMatch(/"dimensions\.parent_dim_no_sql" does not match any of the allowed types/); + expect(e.toString()).toMatch(/"dimensions\.child_dim_no_type" does not match any of the allowed types/); + expect(e.toString()).toMatch(/"dimensions\.child_dim_bad_type" does not match any of the allowed types/); + expect(e.toString()).toMatch(/"dimensions\.child_dim_no_sql" does not match any of the allowed types/); + } + }); }); }); From 01f377fc2a41b28e97e6d6eb1cc599643047d60e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 27 Mar 2025 23:05:34 +0200 Subject: [PATCH 19/22] update joi to the latest --- packages/cubejs-api-gateway/package.json | 2 +- packages/cubejs-schema-compiler/package.json | 2 +- packages/cubejs-server-core/package.json | 2 +- yarn.lock | 36 +++++++++++++------- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index c01cdc0139365..f004627c8ef6f 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -39,7 +39,7 @@ "graphql-tag": "^2.12.6", "http-proxy-middleware": "^3.0.0", "inflection": "^1.12.0", - "joi": "^17.8.3", + "joi": "^17.13.3", "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.4", "moment": "^2.24.0", diff --git a/packages/cubejs-schema-compiler/package.json b/packages/cubejs-schema-compiler/package.json index a24829cee22e3..97f4ecf8f5519 100644 --- a/packages/cubejs-schema-compiler/package.json +++ b/packages/cubejs-schema-compiler/package.json @@ -47,7 +47,7 @@ "cron-parser": "^4.9.0", "humps": "^2.0.1", "inflection": "^1.12.0", - "joi": "^17.8.3", + "joi": "^17.13.3", "js-yaml": "^4.1.0", "lru-cache": "^5.1.1", "moment-timezone": "^0.5.46", diff --git a/packages/cubejs-server-core/package.json b/packages/cubejs-server-core/package.json index a6c3c2a297bce..05e7b05037c05 100644 --- a/packages/cubejs-server-core/package.json +++ b/packages/cubejs-server-core/package.json @@ -43,7 +43,7 @@ "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-docker": "^2.1.1", - "joi": "^17.8.3", + "joi": "^17.13.3", "jsonwebtoken": "^9.0.2", "lodash.clonedeep": "^4.5.0", "lru-cache": "^5.1.1", diff --git a/yarn.lock b/yarn.lock index e505e452c6fe3..35111602228d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5616,7 +5616,12 @@ resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.2.1.tgz#9551142a1980503752536b5050fd99f4a7f13b17" integrity sha512-gfta+H8aziZsm8pZa0vj04KO6biEiisppNgA1kbJvFrrWu9Vm7eaUEy76DIxsuTaWvti5fkJVhllWc6ZTE+Mdw== -"@hapi/topo@^5.0.0": +"@hapi/hoek@^9.3.0": + version "9.3.0" + resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" + integrity sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ== + +"@hapi/topo@^5.0.0", "@hapi/topo@^5.1.0": version "5.1.0" resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-5.1.0.tgz#dc448e332c6c6e37a4dc02fd84ba8d44b9afb012" integrity sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg== @@ -8471,6 +8476,13 @@ dependencies: "@hapi/hoek" "^9.0.0" +"@sideway/address@^4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.5.tgz#4bc149a0076623ced99ca8208ba780d65a99b9d5" + integrity sha512-IqO/DUQHUkPeixNQ8n0JA6102hT9CmaljNTPmQ1u8MEhBo/R4Q8eKLN/vGZxuebwOroDB4cbpjheD4+/sKFK4Q== + dependencies: + "@hapi/hoek" "^9.0.0" + "@sideway/formula@^3.0.1": version "3.0.1" resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.1.tgz#80fcbcbaf7ce031e0ef2dd29b1bfc7c3f583611f" @@ -19699,21 +19711,21 @@ jmespath@0.16.0: resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.16.0.tgz#b15b0a85dfd4d930d43e69ed605943c802785076" integrity sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw== -joi@^17.4.0: - version "17.11.0" - resolved "https://registry.yarnpkg.com/joi/-/joi-17.11.0.tgz#aa9da753578ec7720e6f0ca2c7046996ed04fc1a" - integrity sha512-NgB+lZLNoqISVy1rZocE9PZI36bL/77ie924Ri43yEvi9GUUMPeyVIr8KdFTMUlby1p0PBYMk9spIxEUQYqrJQ== +joi@^17.13.3: + version "17.13.3" + resolved "https://registry.yarnpkg.com/joi/-/joi-17.13.3.tgz#0f5cc1169c999b30d344366d384b12d92558bcec" + integrity sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA== dependencies: - "@hapi/hoek" "^9.0.0" - "@hapi/topo" "^5.0.0" - "@sideway/address" "^4.1.3" + "@hapi/hoek" "^9.3.0" + "@hapi/topo" "^5.1.0" + "@sideway/address" "^4.1.5" "@sideway/formula" "^3.0.1" "@sideway/pinpoint" "^2.0.0" -joi@^17.8.3: - version "17.8.3" - resolved "https://registry.yarnpkg.com/joi/-/joi-17.8.3.tgz#d772fe27a87a5cda21aace5cf11eee8671ca7e6f" - integrity sha512-q5Fn6Tj/jR8PfrLrx4fpGH4v9qM6o+vDUfD4/3vxxyg34OmKcNqYZ1qn2mpLza96S8tL0p0rIw2gOZX+/cTg9w== +joi@^17.4.0: + version "17.11.0" + resolved "https://registry.yarnpkg.com/joi/-/joi-17.11.0.tgz#aa9da753578ec7720e6f0ca2c7046996ed04fc1a" + integrity sha512-NgB+lZLNoqISVy1rZocE9PZI36bL/77ie924Ri43yEvi9GUUMPeyVIr8KdFTMUlby1p0PBYMk9spIxEUQYqrJQ== dependencies: "@hapi/hoek" "^9.0.0" "@hapi/topo" "^5.0.0" From 67c12efd1493062590994aa02d6ca9cf3e4649e3 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 28 Mar 2025 12:28:54 +0200 Subject: [PATCH 20/22] fix tests --- packages/cubejs-schema-compiler/test/unit/schema.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index 025f3ad2ea5df..d58de5a87a02b 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -1010,7 +1010,7 @@ describe('Schema Testing', () => { expect(cubeB.sql).toBeFalsy(); }); - it('throws error for member without type in cubeB extending cubeA', async () => { + it('throws errors for invalid members in both cubes (parent and child)', async () => { const cubes = fs.readFileSync( path.join(process.cwd(), '/test/unit/fixtures/invalid_cubes.yaml'), 'utf8' From ef59eb714db96ee46398faeaa67fcd78ff71b4b4 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 28 Mar 2025 13:55:31 +0200 Subject: [PATCH 21/22] add tests for pre-aggs --- .../test/unit/fixtures/validate_preaggs.js | 98 +++++ .../test/unit/fixtures/validate_preaggs.yml | 65 ++++ .../test/unit/schema.test.ts | 338 ++++++++++-------- 3 files changed, 361 insertions(+), 140 deletions(-) create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.js create mode 100644 packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.yml diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.js b/packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.js new file mode 100644 index 0000000000000..342d63cb38a45 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.js @@ -0,0 +1,98 @@ +cube('validate_preaggs', { + sql_table: 'public.orders', + + dimensions: { + id: { + sql: 'id', + type: 'number', + primary_key: true, + }, + + status: { + sql: 'status', + type: 'string', + }, + + created_at: { + sql: 'created_at', + type: 'time', + }, + + completed_at: { + sql: 'completed_at', + type: 'time', + }, + }, + + measures: { + count: { + type: 'count', + }, + }, + + hierarchies: { + hello: { + title: 'World', + levels: [status], + }, + }, + + preAggregations: { + autoRollupFail: { + type: 'autoRollup', + maxPreAggregations: 'string_instead_of_number', + }, + + originalSqlFail: { + type: 'originalSql', + partitionGranularity: 'invalid_partition_granularity', + }, + + originalSqlFail2: { + type: 'originalSql', + partitionGranularity: 'day', + uniqueKeyColumns: 'not_an_array', + }, + + rollupJoinFail: { + type: 'rollupJoin', + partitionGranularity: 'day', + // no rollups + }, + + rollupLambdaFail: { + type: 'rollupLambda', + partitionGranularity: 'day', + granularity: 'day', + time_dimension: created_at, + rollups: not_a_func, + }, + + rollupFail: { + type: 'rollup', + measures: [CUBE.count], + timeDimension: [CUBE.created_at], + granularity: 'day', + partitionGranularity: 'month', + }, + + rollupFail2: { + type: 'rollup', + granularity: `day`, + partitionGranularity: `month`, + dimensions: [CUBE.created_at], + measures: [CUBE.count], + timeDimensions: 'created_at', + }, + + // TODO: implement check for strings in dimensions/measures/timeDimension + // rollupFail3: { + // type: 'rollup', + // granularity: `day`, + // partitionGranularity: `month`, + // dimensions: ['created_at'], + // measures: ['count'], + // timeDimension: 'created_at', + // }, + }, +}); diff --git a/packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.yml b/packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.yml new file mode 100644 index 0000000000000..ef6d549e11c7a --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/fixtures/validate_preaggs.yml @@ -0,0 +1,65 @@ +cubes: + - name: validate_preaggs + sql: SELECT * FROM order_users; + + measures: + - name: count + sql: id + type: count + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: name + sql: name + type: string + + - name: created_at + sql: created_at + type: time + + pre_aggregations: + - name: autoRollupFail + type: autoRollup + maxPreAggregations: string_instead_of_number + + - name: originalSqlFail + type: originalSql + partition_granularity: invalid_partition_granularity + + - name: originalSqlFail2 + type: originalSql + partition_granularity: month + uniqueKeyColumns: not_an_array + + - name: rollupJoinFail + type: rollupJoin + partition_granularity: month + # no rollups + + - name: rollupLambdaFail + type: rollupLambda + partition_granularity: month + granularity: day + time_dimension: created_at + rollups: not_a_func + + - name: rollupFail + type: rollup + measures: + - CUBE.count + time_dimension: # as array + - CUBE.created_at + granularity: day + partition_granularity: month + refresh_key: + every: 1 hour + scheduled_refresh: true + + - name: rollupFail2 + type: rollup + measures: + - CUBE.count + time_dimensions: CUBE.created_at # not an array diff --git a/packages/cubejs-schema-compiler/test/unit/schema.test.ts b/packages/cubejs-schema-compiler/test/unit/schema.test.ts index d58de5a87a02b..061aa51dd58ae 100644 --- a/packages/cubejs-schema-compiler/test/unit/schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/schema.test.ts @@ -57,126 +57,127 @@ describe('Schema Testing', () => { return { compiler, cubeEvaluator }; }; - it('valid schemas', async () => { - const { cubeEvaluator } = await schemaCompile(); - - expect(cubeEvaluator.preAggregationsForCube('CubeA')).toEqual({ - main: { - external: false, - scheduledRefresh: true, - timeDimensionReference: expect.any(Function), - partitionGranularity: 'month', - type: 'originalSql', - refreshRangeStart: { - sql: expect.any(Function), - }, - refreshRangeEnd: { - sql: expect.any(Function), - }, - }, - countCreatedAt: { - external: true, - scheduledRefresh: true, - granularity: 'day', - measureReferences: expect.any(Function), - timeDimensionReference: expect.any(Function), - partitionGranularity: 'month', - type: 'rollup', - refreshRangeStart: { - sql: expect.any(Function), + describe('Cubes validations', () => { + it('valid schemas', async () => { + const { cubeEvaluator } = await schemaCompile(); + + expect(cubeEvaluator.preAggregationsForCube('CubeA')).toEqual({ + main: { + external: false, + scheduledRefresh: true, + timeDimensionReference: expect.any(Function), + partitionGranularity: 'month', + type: 'originalSql', + refreshRangeStart: { + sql: expect.any(Function), + }, + refreshRangeEnd: { + sql: expect.any(Function), + }, }, - refreshRangeEnd: { - sql: expect.any(Function), + countCreatedAt: { + external: true, + scheduledRefresh: true, + granularity: 'day', + measureReferences: expect.any(Function), + timeDimensionReference: expect.any(Function), + partitionGranularity: 'month', + type: 'rollup', + refreshRangeStart: { + sql: expect.any(Function), + }, + refreshRangeEnd: { + sql: expect.any(Function), + }, }, - }, - countCreatedAtWithoutReferences: { + countCreatedAtWithoutReferences: { // because preview - external: true, - scheduledRefresh: true, - granularity: 'day', - measureReferences: expect.any(Function), - timeDimensionReference: expect.any(Function), - segmentReferences: expect.any(Function), - dimensionReferences: expect.any(Function), - partitionGranularity: 'month', - type: 'rollup', - refreshRangeStart: { - sql: expect.any(Function), - }, - refreshRangeEnd: { - sql: expect.any(Function), - }, - } + external: true, + scheduledRefresh: true, + granularity: 'day', + measureReferences: expect.any(Function), + timeDimensionReference: expect.any(Function), + segmentReferences: expect.any(Function), + dimensionReferences: expect.any(Function), + partitionGranularity: 'month', + type: 'rollup', + refreshRangeStart: { + sql: expect.any(Function), + }, + refreshRangeEnd: { + sql: expect.any(Function), + }, + } + }); }); - }); - it('valid schemas (preview flags)', async () => { - process.env.CUBEJS_EXTERNAL_DEFAULT = 'true'; - process.env.CUBEJS_SCHEDULED_REFRESH_DEFAULT = 'true'; + it('valid schemas (preview flags)', async () => { + process.env.CUBEJS_EXTERNAL_DEFAULT = 'true'; + process.env.CUBEJS_SCHEDULED_REFRESH_DEFAULT = 'true'; - const { cubeEvaluator } = await schemaCompile(); + const { cubeEvaluator } = await schemaCompile(); - delete process.env.CUBEJS_EXTERNAL_DEFAULT; - delete process.env.CUBEJS_SCHEDULED_REFRESH_DEFAULT; + delete process.env.CUBEJS_EXTERNAL_DEFAULT; + delete process.env.CUBEJS_SCHEDULED_REFRESH_DEFAULT; - expect(cubeEvaluator.preAggregationsForCube('CubeA')).toEqual({ - main: { - external: false, - scheduledRefresh: true, - timeDimensionReference: expect.any(Function), - partitionGranularity: 'month', - type: 'originalSql', - refreshRangeStart: { - sql: expect.any(Function), - }, - refreshRangeEnd: { - sql: expect.any(Function), + expect(cubeEvaluator.preAggregationsForCube('CubeA')).toEqual({ + main: { + external: false, + scheduledRefresh: true, + timeDimensionReference: expect.any(Function), + partitionGranularity: 'month', + type: 'originalSql', + refreshRangeStart: { + sql: expect.any(Function), + }, + refreshRangeEnd: { + sql: expect.any(Function), + }, }, - }, - countCreatedAt: { + countCreatedAt: { // because preview - external: true, - scheduledRefresh: true, - granularity: 'day', - measureReferences: expect.any(Function), - timeDimensionReference: expect.any(Function), - partitionGranularity: 'month', - type: 'rollup', - refreshRangeStart: { - sql: expect.any(Function), - }, - refreshRangeEnd: { - sql: expect.any(Function), + external: true, + scheduledRefresh: true, + granularity: 'day', + measureReferences: expect.any(Function), + timeDimensionReference: expect.any(Function), + partitionGranularity: 'month', + type: 'rollup', + refreshRangeStart: { + sql: expect.any(Function), + }, + refreshRangeEnd: { + sql: expect.any(Function), + }, }, - }, - countCreatedAtWithoutReferences: { + countCreatedAtWithoutReferences: { // because preview - external: true, - scheduledRefresh: true, - granularity: 'day', - measureReferences: expect.any(Function), - segmentReferences: expect.any(Function), - dimensionReferences: expect.any(Function), - timeDimensionReference: expect.any(Function), - partitionGranularity: 'month', - type: 'rollup', - refreshRangeStart: { - sql: expect.any(Function), - }, - refreshRangeEnd: { - sql: expect.any(Function), - }, - } + external: true, + scheduledRefresh: true, + granularity: 'day', + measureReferences: expect.any(Function), + segmentReferences: expect.any(Function), + dimensionReferences: expect.any(Function), + timeDimensionReference: expect.any(Function), + partitionGranularity: 'month', + type: 'rollup', + refreshRangeStart: { + sql: expect.any(Function), + }, + refreshRangeEnd: { + sql: expect.any(Function), + }, + } + }); }); - }); - it('invalid schema', async () => { - const logger = jest.fn(); + it('invalid schema', async () => { + const logger = jest.fn(); - const { compiler } = prepareJsCompiler( - createCubeSchema({ - name: 'CubeA', - preAggregations: ` + const { compiler } = prepareJsCompiler( + createCubeSchema({ + name: 'CubeA', + preAggregations: ` main: { type: 'originalSql', timeDimension: createdAt, @@ -195,25 +196,102 @@ describe('Schema Testing', () => { } }, ` - }), - { - omitErrors: true, - errorReport: { - logger, + }), + { + omitErrors: true, + errorReport: { + logger, + } } + ); + + await compiler.compile(); + compiler.throwIfAnyErrors(); + + expect(logger.mock.calls.length).toEqual(2); + expect(logger.mock.calls[0]).toEqual([ + 'You specified both buildRangeStart and refreshRangeStart, buildRangeStart will be used.' + ]); + expect(logger.mock.calls[1]).toEqual([ + 'You specified both buildRangeEnd and refreshRangeEnd, buildRangeEnd will be used.' + ]); + }); + + it('throws an error on duplicate member names', async () => { + const orders = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/orders_dup_members.js'), + 'utf8' + ); + + const { compiler } = prepareCompiler([ + { + content: orders, + fileName: 'orders.js', + }, + ]); + + try { + await compiler.compile(); + } catch (e: any) { + expect(e.toString()).toMatch(/status defined more than once/); } - ); + }); - await compiler.compile(); - compiler.throwIfAnyErrors(); + it('throws errors for invalid pre-aggregations in yaml data model', async () => { + const cubes = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/validate_preaggs.yml'), + 'utf8' + ); + const { compiler } = prepareCompiler([ + { + content: cubes, + fileName: 'validate_preaggs.yml', + }, + ]); - expect(logger.mock.calls.length).toEqual(2); - expect(logger.mock.calls[0]).toEqual([ - 'You specified both buildRangeStart and refreshRangeStart, buildRangeStart will be used.' - ]); - expect(logger.mock.calls[1]).toEqual([ - 'You specified both buildRangeEnd and refreshRangeEnd, buildRangeEnd will be used.' - ]); + try { + await compiler.compile(); + throw new Error('should throw earlier'); + } catch (e: any) { + expect(e.toString()).toMatch(/"preAggregations\.autoRollupFail\.maxPreAggregations" must be a number/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail\.partitionGranularity" must be one of/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail\.timeDimension" is required/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail2\.uniqueKeyColumns" must be an array/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail2\.timeDimension" is required/); + expect(e.toString()).toMatch(/"preAggregations\.rollupJoinFail" does not match any of the allowed types/); + expect(e.toString()).toMatch(/"preAggregations\.rollupLambdaFail\.partitionGranularity" is not allowed/); + // TODO preAggregations.rollupFail.timeDimension - should catch that it is an array, currently not catching + expect(e.toString()).toMatch(/"preAggregations\.rollupFail2\.timeDimensions" must be an array/); + } + }); + + it('throws errors for invalid pre-aggregations in js data model', async () => { + const cubes = fs.readFileSync( + path.join(process.cwd(), '/test/unit/fixtures/validate_preaggs.js'), + 'utf8' + ); + const { compiler } = prepareCompiler([ + { + content: cubes, + fileName: 'validate_preaggs.js', + }, + ]); + + try { + await compiler.compile(); + throw new Error('should throw earlier'); + } catch (e: any) { + expect(e.toString()).toMatch(/"preAggregations\.autoRollupFail\.maxPreAggregations" must be a number/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail\.partitionGranularity" must be one of/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail\.timeDimension" is required/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail2\.uniqueKeyColumns" must be an array/); + expect(e.toString()).toMatch(/"preAggregations\.originalSqlFail2\.timeDimension" is required/); + expect(e.toString()).toMatch(/"preAggregations\.rollupJoinFail" does not match any of the allowed types/); + expect(e.toString()).toMatch(/"preAggregations\.rollupLambdaFail\.partitionGranularity" is not allowed/); + // TODO preAggregations.rollupFail.timeDimension - should catch that it is an array, currently not catching + expect(e.toString()).toMatch(/"preAggregations\.rollupFail2\.timeDimensions" must be an array/); + } + }); }); it('visibility modifier', async () => { @@ -372,26 +450,6 @@ describe('Schema Testing', () => { }); }); - it('throws an error on duplicate member names', async () => { - const orders = fs.readFileSync( - path.join(process.cwd(), '/test/unit/fixtures/orders_dup_members.js'), - 'utf8' - ); - - const { compiler } = prepareCompiler([ - { - content: orders, - fileName: 'orders.js', - }, - ]); - - try { - await compiler.compile(); - } catch (e: any) { - expect(e.toString()).toMatch(/status defined more than once/); - } - }); - describe('Access Policies', () => { it('valid schema with accessPolicy', async () => { const { compiler } = prepareJsCompiler([ From dbf8633b6a1ebebaf44fc5c096c48addc8463378 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Fri, 28 Mar 2025 13:59:48 +0200 Subject: [PATCH 22/22] fix test --- packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 2b926a0ec6e24..2acd7fa25ce1a 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -108,7 +108,7 @@ describe('Yaml Schema Testing', () => { throw new Error('compile must return an error'); } catch (e: any) { - expect(e.message).toContain('Users cube: (title = null) must be a string'); + expect(e.message).toContain('Users cube: "title" must be a string'); } });