Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(schema-compiler): Allow passing function that returns an array of time dimensions as pre-aggregation timeDimensions property #9125

Closed
wants to merge 1 commit into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
feat(schema-compiler): Allow passing function that returns an array o…
…f time dimensions as pre-aggregation timeDimensions property
KSDaemon committed Jan 27, 2025
commit 12a65ce46156a8a836617ca47976d8490d3eee61
Original file line number Diff line number Diff line change
@@ -696,14 +696,22 @@ export class CubeEvaluator extends CubeSymbols {
dimension: this.evaluateReferences(cube, aggregation.timeDimensionReference),
granularity: aggregation.granularity
});
} else if (aggregation.timeDimensionReferences) {
} else if (Array.isArray(aggregation.timeDimensionReferences)) {
// eslint-disable-next-line guard-for-in
for (const timeDimensionReference of aggregation.timeDimensionReferences) {
timeDimensions.push({
dimension: this.evaluateReferences(cube, timeDimensionReference.dimension),
granularity: timeDimensionReference.granularity
});
}
} else if (aggregation.timeDimensionReferences) {
const evaluatedRefs: any[] = this.evaluateReferences(cube, aggregation.timeDimensionReferences, { returnRaw: true });
for (const timeDimensionReference of evaluatedRefs) {
timeDimensions.push({
dimension: timeDimensionReference.dimension.toString(),
granularity: timeDimensionReference.granularity
});
}
}

return {
5 changes: 5 additions & 0 deletions packages/cubejs-schema-compiler/src/compiler/CubeSymbols.js
Original file line number Diff line number Diff line change
@@ -515,6 +515,11 @@ export class CubeSymbols {
cubeAliasFn: (cube) => cubeEvaluator.pathFromArray(fullPath(cubeEvaluator.joinHints(), [cube])),
collectJoinHints: options.collectJoinHints,
});

if (options.returnRaw) {
return arrayOrSingle;
}

if (!Array.isArray(arrayOrSingle)) {
return arrayOrSingle.toString();
}
11 changes: 7 additions & 4 deletions packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts
Original file line number Diff line number Diff line change
@@ -473,10 +473,13 @@ const RollUpSchema = condition(
// Rollup with multiple time dimensions
inherit(BasePreAggregation, {
type: Joi.any().valid('rollup').required(),
timeDimensions: Joi.array().items(Joi.object().keys({
dimension: Joi.func(),
granularity: GranularitySchema,
})),
timeDimensions: Joi.alternatives().try(
Joi.array().items(Joi.object().keys({
dimension: Joi.func(),
granularity: GranularitySchema,
})),
Joi.func(),
),
allowNonStrictDateRangeMatch: Joi.bool(),
measures: Joi.func(),
dimensions: Joi.func(),
162 changes: 162 additions & 0 deletions packages/cubejs-schema-compiler/test/unit/pre-aggregations.test.ts
Original file line number Diff line number Diff line change
@@ -92,6 +92,168 @@ describe('pre-aggregations', () => {
expect(cubeEvaluator.cubeFromPath('Orders').preAggregations.ordersRollupJoin.scheduledRefresh).toEqual(undefined);
});

it('Rollup with pre-agg with hardcoded multiple time dimensions', async () => {
const { compiler, cubeEvaluator, joinGraph } = prepareCompiler(
`
cube(\`Users\`, {
sql: \`SELECT * FROM public.users\`,

preAggregations: {
staticMultiple: {
dimensions: [CUBE.status],
measures: [CUBE.count],
timeDimensions: [
{ dimension: CUBE.createdAt, granularity: \`day\` },
{ dimension: CUBE.modifiedAt, granularity: \`day\` },
]
}
},

measures: {
count: {
type: \`count\`,
},
},

dimensions: {
id: {
sql: \`id\`,
type: \`string\`,
primaryKey: true,
},

name: {
sql: \`name\`,
type: \`string\`,
},

userId: {
sql: \`user_id\`,
type: \`number\`,
},
status: {
sql: \`status\`,
type: \`string\`,
},

createdAt: {
type: \`time\`,
sql: \`created_at\`
},

modifiedAt: {
type: \`time\`,
sql: \`modified_at\`
}
},
});
`
);

await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
dimensions: ['Users.status'],
measures: ['Users.count'],
timeDimensions: [{
dimension: 'Users.createdAt',
dateRange: ['2023-01-20', '2024-01-20'],
granularity: 'day'
}]
});

const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
expect(queryAndParams[0].includes('undefined')).toBeFalsy();
expect(queryAndParams[0].includes('pre_aggregations')).toBeTruthy();

expect(preAggregationsDescription.length).toEqual(1);
expect(preAggregationsDescription[0].preAggregationId).toEqual('Users.staticMultiple');
});

it('Rollup with pre-agg with dynamic multiple time dimensions', async () => {
const { compiler, cubeEvaluator, joinGraph } = prepareCompiler(
`
cube(\`Users\`, {
sql: \`SELECT * FROM public.users\`,

preAggregations: {
dynamicMultiple: {
dimensions: [CUBE.status],
measures: [CUBE.count],
timeDimensions: (CUBE) => [
{dimension: CUBE.createdAt, granularity: 'day'},
{dimension: CUBE.modifiedAt, granularity: 'day'},
]
},
},

measures: {
count: {
type: \`count\`,
},
},

dimensions: {
id: {
sql: \`id\`,
type: \`string\`,
primaryKey: true,
},

name: {
sql: \`name\`,
type: \`string\`,
},

userId: {
sql: \`user_id\`,
type: \`number\`,
},
status: {
sql: \`status\`,
type: \`string\`,
},

createdAt: {
type: \`time\`,
sql: \`created_at\`
},

modifiedAt: {
type: \`time\`,
sql: \`modified_at\`
}
},
});
`
);

await compiler.compile();

const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, {
dimensions: ['Users.status'],
measures: ['Users.count'],
timeDimensions: [{
dimension: 'Users.createdAt',
dateRange: ['2023-01-20', '2024-01-20'],
granularity: 'day'
}]
});

const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription();

const queryAndParams = query.buildSqlAndParams();
console.log(queryAndParams);
expect(queryAndParams[0].includes('undefined')).toBeFalsy();
expect(queryAndParams[0].includes('pre_aggregations')).toBeTruthy();

expect(preAggregationsDescription.length).toEqual(1);
expect(preAggregationsDescription[0].preAggregationId).toEqual('Users.dynamicMultiple');
});

it('query rollupLambda', async () => {
const { compiler, cubeEvaluator, joinGraph } = prepareCompiler(
`

Unchanged files with check annotations Beta

// TODO switch parsing to microseconds
if timestamp.and_utc().timestamp_millis() > (((1i64) << 62) / 1_000_000) {
builder.append_null()?;
} else if let Some(nanos) = timestamp.timestamp_nanos_opt() {

Check warning on line 1132 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

GitHub Actions / unit (20.x, 3.11)

use of deprecated method `chrono::NaiveDateTime::timestamp_nanos_opt`: use `.and_utc().timestamp_nanos_opt()` instead

Check warning on line 1132 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

GitHub Actions / build

use of deprecated method `chrono::NaiveDateTime::timestamp_nanos_opt`: use `.and_utc().timestamp_nanos_opt()` instead

Check warning on line 1132 in rust/cubesql/cubesql/src/compile/engine/df/scan.rs

GitHub Actions / unit (22.x, 3.11)

use of deprecated method `chrono::NaiveDateTime::timestamp_nanos_opt`: use `.and_utc().timestamp_nanos_opt()` instead
builder.append_value(nanos)?;
} else {
log::error!(
let secs = duration.num_seconds();
let nanosecs = duration.num_nanoseconds().unwrap_or(0) - secs * 1_000_000_000;
let timestamp = NaiveDateTime::from_timestamp_opt(secs, nanosecs as u32)

Check warning on line 1672 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (20.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 1672 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / build

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 1672 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (22.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead
.unwrap_or_else(|| panic!("Invalid secs {} nanosecs {}", secs, nanosecs));
// chrono's strftime is missing quarter format, as such a workaround is required
macro_rules! generate_series_helper_date32 {
($CURRENT:ident, $STEP:ident, $PRIMITIVE_TYPE: ident) => {
let current_dt = NaiveDateTime::from_timestamp_opt(($CURRENT as i64) * 86400, 0)

Check warning on line 2319 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (20.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 2319 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / build

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 2319 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (22.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead
.ok_or_else(|| {
DataFusionError::Execution(format!(
"Cannot convert date to NaiveDateTime: {}",
))
})?;
let res = date_addsub_month_day_nano(current_dt, $STEP, true)?;
$CURRENT = (res.timestamp() / 86400) as $PRIMITIVE_TYPE;

Check warning on line 2327 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (20.x, 3.11)

use of deprecated method `chrono::NaiveDateTime::timestamp`: use `.and_utc().timestamp()` instead

Check warning on line 2327 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / build

use of deprecated method `chrono::NaiveDateTime::timestamp`: use `.and_utc().timestamp()` instead

Check warning on line 2327 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (22.x, 3.11)

use of deprecated method `chrono::NaiveDateTime::timestamp`: use `.and_utc().timestamp()` instead
};
}
macro_rules! generate_series_helper_timestamp {
($CURRENT:ident, $STEP:ident, $PRIMITIVE_TYPE: ident) => {
let current_dt = NaiveDateTime::from_timestamp_opt(

Check warning on line 2333 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (20.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 2333 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / build

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 2333 in rust/cubesql/cubesql/src/compile/engine/udf/common.rs

GitHub Actions / unit (22.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead
($CURRENT as i64) / 1_000_000_000,
($CURRENT % 1_000_000_000) as u32,
)
};
let ts_seconds = *ts / 1_000_000_000;
let ts_nanos = (*ts % 1_000_000_000) as u32;
let dt = NaiveDateTime::from_timestamp_opt(ts_seconds, ts_nanos).map(|dt| Some(dt));

Check warning on line 4817 in rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs

GitHub Actions / unit (20.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 4817 in rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs

GitHub Actions / build

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 4817 in rust/cubesql/cubesql/src/compile/rewrite/rules/filters.rs

GitHub Actions / unit (22.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead
return dt;
};
return Some(false);
}
let seconds = ns / ns_in_seconds;
let dt = NaiveDateTime::from_timestamp_opt(seconds, 0)?;

Check warning on line 460 in rust/cubesql/cubesql/src/compile/rewrite/rules/utils.rs

GitHub Actions / unit (20.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 460 in rust/cubesql/cubesql/src/compile/rewrite/rules/utils.rs

GitHub Actions / build

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead

Check warning on line 460 in rust/cubesql/cubesql/src/compile/rewrite/rules/utils.rs

GitHub Actions / unit (22.x, 3.11)

use of deprecated associated function `chrono::NaiveDateTime::from_timestamp_opt`: use `DateTime::from_timestamp` instead
let is_minute_trunced = |dt: NaiveDateTime| dt.second() == 0;
let is_hour_trunced = |dt| is_minute_trunced(dt) && dt.minute() == 0;