diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-interpolated-cube-refs.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-interpolated-cube-refs.test.ts new file mode 100644 index 0000000000000..fdd1a8e7a333f --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-interpolated-cube-refs.test.ts @@ -0,0 +1,122 @@ +import { prepareJsCompiler } from './PrepareCompiler'; +import { PostgresQuery } from '../../src/adapter/PostgresQuery'; + +/** + * Pre-aggregation references built by interpolating the cube itself + * (`` (CUBE) => `${CUBE}.issued_date` ``) instead of a member. The cube + * stringifies to its name, so such a reference names the member as text; both + * planners have to resolve it to the same member. + */ +describe('pre-aggregation references interpolating the cube', () => { + const model = (preAggregations: string) => ` + const getCubeFields = (cube, names) => names.map((name) => cube[name]); + + cube('invoices', { + sql: 'SELECT * FROM invoices', + + measures: { + count: { type: 'count' }, + total: { sql: 'amount', type: 'sum' }, + }, + + dimensions: { + id: { sql: 'id', type: 'number', primaryKey: true }, + org_id: { sql: 'org_id', type: 'string' }, + issued_date: { sql: 'issued_date', type: 'time' }, + payment_received_date: { sql: 'payment_received_date', type: 'time' }, + }, + + preAggregations: ${preAggregations}, + }); + `; + + const preAggregationsWithInterpolatedTimeDimension = `{ + by_org_and_issued_date: { + type: 'rollup', + measures: (CUBE) => getCubeFields(CUBE, ['count', 'total']), + dimensions: (CUBE) => getCubeFields(CUBE, ['org_id']), + timeDimension: (CUBE) => \`\${CUBE}.issued_date\`, + granularity: 'day', + partitionGranularity: 'month', + }, + by_org_and_payment_received_date: { + type: 'rollup', + measures: (CUBE) => getCubeFields(CUBE, ['count', 'total']), + dimensions: (CUBE) => getCubeFields(CUBE, ['org_id']), + timeDimension: (CUBE) => \`\${CUBE}.payment_received_date\`, + granularity: 'day', + partitionGranularity: 'month', + }, + by_org_all_time: { + type: 'rollup', + measures: (CUBE) => getCubeFields(CUBE, ['count', 'total']), + dimensions: (CUBE) => getCubeFields(CUBE, ['org_id']), + }, + }`; + + const preAggregationsWithInterpolatedMembers = `{ + by_org_and_issued_date: { + type: 'rollup', + measures: (CUBE) => [\`\${CUBE}.count\`, \`\${CUBE}.total\`], + dimensions: (CUBE) => [\`\${CUBE}.org_id\`], + timeDimension: (CUBE) => \`\${CUBE}.issued_date\`, + granularity: 'day', + partitionGranularity: 'month', + }, + }`; + + const preAggregationWithGranularitySuffix = `{ + by_org_and_issued_date: { + type: 'rollup', + measures: (CUBE) => getCubeFields(CUBE, ['count']), + timeDimension: (CUBE) => \`\${CUBE}.issued_date_day\`, + granularity: 'day', + partitionGranularity: 'month', + }, + }`; + + async function buildQuery(preAggregations: string, useNativeSqlPlanner: boolean) { + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(model(preAggregations)); + await compiler.compile(); + + return new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + measures: ['invoices.count'], + dimensions: ['invoices.org_id'], + timeDimensions: [{ + dimension: 'invoices.issued_date', + granularity: 'day', + dateRange: ['2020-01-01', '2020-03-31'], + }], + timezone: 'UTC', + useNativeSqlPlanner, + }); + } + + for (const useNativeSqlPlanner of [false, true]) { + const planner = useNativeSqlPlanner ? 'tesseract' : 'legacy'; + + it(`resolves an interpolated time dimension (${planner})`, async () => { + const query = await buildQuery(preAggregationsWithInterpolatedTimeDimension, useNativeSqlPlanner); + query.buildSqlAndParams(); + + const descriptions: any = query.preAggregations?.preAggregationsDescription(); + expect(descriptions.map(d => d.preAggregationId)).toEqual(['invoices.by_org_and_issued_date']); + }); + + it(`resolves interpolated measure and dimension references (${planner})`, async () => { + const query = await buildQuery(preAggregationsWithInterpolatedMembers, useNativeSqlPlanner); + query.buildSqlAndParams(); + + const descriptions: any = query.preAggregations?.preAggregationsDescription(); + expect(descriptions.map(d => d.preAggregationId)).toEqual(['invoices.by_org_and_issued_date']); + }); + + it(`reports the member an interpolated reference names when it does not exist (${planner})`, async () => { + const query = await buildQuery(preAggregationWithGranularitySuffix, useNativeSqlPlanner); + + expect(() => query.buildSqlAndParams()).toThrow( + /'issued_date_day' not found for path 'invoices.issued_date_day'/ + ); + }); + } +}); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs index ed31b21fc2b07..df68099ec8e2e 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs @@ -13,11 +13,17 @@ use crate::planner::multi_fact_join_groups::{MeasuresJoinHints, MultiFactJoinGro use crate::planner::planners::JoinPlanner; use crate::planner::planners::ResolvedJoinItem; use crate::planner::state::State; +use crate::planner::Compiler; use crate::planner::GranularityHelper; use crate::planner::MemberSymbol; +use crate::planner::SqlCall; +use crate::planner::SqlCallReference; +use crate::planner::SymbolPath; +use crate::planner::SymbolPathType; use crate::planner::TimeDimensionSymbol; use crate::utils::debug::DebugSql; use cubenativeutils::CubeError; +use cubenativeutils::CubeErrorCauseType; use itertools::Itertools; use std::collections::HashMap; use std::fmt::Debug; @@ -112,19 +118,14 @@ impl PreAggregationsCompiler { } let measures = if let Some(refs) = description.measure_references()? { - Self::symbols_from_ref( - self.query_tools.clone(), - &name.cube_name, - refs, - Self::check_is_measure, - )? + Self::symbols_from_ref(self.query_tools.clone(), name, refs, Self::check_is_measure)? } else { Vec::new() }; let dimensions = if let Some(refs) = description.dimension_references()? { Self::symbols_from_ref( self.query_tools.clone(), - &name.cube_name, + name, refs, Self::check_is_dimension, )? @@ -192,12 +193,7 @@ impl PreAggregationsCompiler { Vec::new() }; let segments = if let Some(refs) = description.segment_references()? { - Self::symbols_from_ref( - self.query_tools.clone(), - &name.cube_name, - refs, - Self::check_is_segment, - )? + Self::symbols_from_ref(self.query_tools.clone(), name, refs, Self::check_is_segment)? } else { Vec::new() }; @@ -553,19 +549,19 @@ impl PreAggregationsCompiler { fn symbols_from_ref Result<(), CubeError>>( query_tools: Rc, - cube_name: &String, + name: &PreAggregationFullName, ref_func: Rc, check_type_fn: F, ) -> Result>, CubeError> { let evaluator_compiler_cell = query_tools.compiler().clone(); let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); - let sql_call = evaluator_compiler.compile_sql_call(cube_name, ref_func)?; - let mut res = Vec::new(); - for symbol in sql_call.get_dependencies().iter() { - check_type_fn(&symbol)?; - res.push(symbol.clone()); + let sql_call = evaluator_compiler.compile_sql_call(&name.cube_name, ref_func)?; + let symbols = + Self::reference_symbols(&query_tools, &mut evaluator_compiler, name, &sql_call)?; + for symbol in symbols.iter() { + check_type_fn(symbol)?; } - Ok(res) + Ok(symbols) } fn time_dimension_symbol_from_ref( @@ -577,24 +573,75 @@ impl PreAggregationsCompiler { let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); let sql_call = evaluator_compiler.compile_sql_call(&name.cube_name, ref_func)?; - let mut symbols = Vec::new(); - - for symbol in sql_call.get_dependencies().into_iter() { - Self::check_is_time_dimension(&symbol)?; - symbols.push(symbol); + let symbols = + Self::reference_symbols(&query_tools, &mut evaluator_compiler, name, &sql_call)?; + for symbol in symbols.iter() { + Self::check_is_time_dimension(symbol)?; } symbols.into_iter().next().ok_or_else(|| { let path = sql_call.debug_sql(true); - let member_name = path.rsplit('.').next().unwrap_or(&path); - - CubeError::user(format!( - "'{}' not found for path '{}' in pre-aggregation '{}.{}'", - member_name, path, name.cube_name, name.name - )) + Self::reference_not_found_error(&path, name) }) } + /// Members a pre-aggregation reference declaration names, in declaration + /// order. An element that interpolated the cube instead of the member is + /// resolved here; an element naming no member at all is an error, so a + /// reference is never silently dropped. + fn reference_symbols( + query_tools: &Rc, + evaluator_compiler: &mut Compiler, + name: &PreAggregationFullName, + sql_call: &SqlCall, + ) -> Result>, CubeError> { + let mut result = Vec::new(); + for item in sql_call.reference_items() { + let symbol = match item { + SqlCallReference::Symbol(symbol) => symbol, + SqlCallReference::Path(path) => { + let full_name = path.join("."); + let symbol_path = + SymbolPath::parse(query_tools.cube_evaluator().clone(), &full_name) + // A path the data model doesn't know is reported with the + // pre-aggregation it came from; anything else (a failure + // reaching the model at all) is passed through as it is. + .map_err(|e| match e.cause { + CubeErrorCauseType::User => { + Self::reference_not_found_error(&full_name, name) + } + _ => e, + })?; + match symbol_path.path_type() { + SymbolPathType::Dimension => { + evaluator_compiler.add_dimension_evaluator_by_path(symbol_path)? + } + SymbolPathType::Measure => { + evaluator_compiler.add_measure_evaluator_by_path(symbol_path)? + } + SymbolPathType::Segment => { + evaluator_compiler.add_segment_evaluator_by_path(symbol_path)? + } + _ => return Err(Self::reference_not_found_error(&full_name, name)), + } + } + SqlCallReference::Unresolved(rendered) => { + return Err(Self::reference_not_found_error(&rendered, name)) + } + }; + result.push(symbol); + } + Ok(result) + } + + fn reference_not_found_error(path: &str, name: &PreAggregationFullName) -> CubeError { + let member_name = path.rsplit('.').next().unwrap_or(path); + CubeError::user(format!( + "'{}' not found for path '{}' in pre-aggregation '{}.{}'", + member_name, path, name.cube_name, name.name + )) + } + fn check_is_measure(symbol: &MemberSymbol) -> Result<(), CubeError> { symbol .as_measure() @@ -640,13 +687,11 @@ mod tests { use crate::test_fixtures::test_utils::TestContext; use indoc::indoc; - fn create_time_dimension_context() -> TestContext { - // `time_dimension: \"{CUBE}.created_at\"` models a JS reference built - // via string interpolation — `(CUBE) => `${CUBE}.created_at``. The JS - // planner resolves it (reference evaluation stringifies `${CUBE}` to - // the cube name), but here `{CUBE}` compiles to a cube reference and - // `.created_at` stays literal text, so the compiled reference has no - // member symbol dependencies. + fn create_reference_context() -> TestContext { + // Template syntax like `\"{CUBE}.created_at\"` models a reference built by + // interpolating the cube itself — `(CUBE) => `${CUBE}.created_at``, where + // the member name arrives as literal text next to a cube reference + // instead of as a member symbol. let schema = MockSchema::from_yaml(indoc! {" cubes: - name: orders @@ -656,36 +701,88 @@ mod tests { type: number sql: id primary_key: true + - name: status + type: string + sql: status - name: created_at type: time sql: created_at + - name: city + type: string + sql: city measures: - name: count type: count + - name: total + type: sum + sql: amount + segments: + - name: completed + sql: \"{CUBE}.status = 'completed'\" pre_aggregations: - - name: working_rollup + - name: symbol_rollup type: rollup measures: - count time_dimension: created_at granularity: day - - name: broken_rollup_unsupported + - name: interpolated_rollup type: rollup measures: - - count + - '{CUBE}.count' + dimensions: + - '{CUBE}.status' + segments: + - '{CUBE}.completed' time_dimension: '{CUBE}.created_at' granularity: day - - name: broken_rollup_no_granularity + - name: interpolated_rollup_no_granularity type: rollup measures: - count time_dimension: '{CUBE}.created_at' + - name: interpolated_rollup_mixed_list + type: rollup + measures: + - '{CUBE}.total' + - count + dimensions: + - status + - '{CUBE}.city' + time_dimension: '{CUBE}.created_at' + granularity: day - name: broken_rollup_granularity_suffix type: rollup measures: - count time_dimension: '{CUBE}.created_at_day' granularity: day + - name: broken_rollup_unknown_measure + type: rollup + measures: + - '{CUBE}.unknown_total' + time_dimension: created_at + granularity: day + - name: interpolated_rollup_granularity_segment + type: rollup + measures: + - count + time_dimension: '{CUBE}.created_at.day' + granularity: day + - name: symbol_rollup_granularity_segment + type: rollup + measures: + - count + time_dimension: created_at.day + granularity: day + - name: broken_rollup_expression_dimension + type: rollup + measures: + - count + dimensions: + - \"{CUBE}.status = 'completed'\" + time_dimension: created_at + granularity: day "}) .unwrap(); TestContext::new(schema).unwrap() @@ -704,8 +801,8 @@ mod tests { #[test] fn test_time_dimension_resolves_to_member_symbol() { - let ctx = create_time_dimension_context(); - let compiled = compile_pre_agg(&ctx, "working_rollup").unwrap(); + let ctx = create_reference_context(); + let compiled = compile_pre_agg(&ctx, "symbol_rollup").unwrap(); assert_eq!(compiled.time_dimensions.len(), 1); assert_eq!( @@ -716,32 +813,86 @@ mod tests { } #[test] - fn test_time_dimension_resolved_to_cube_ref_returns_error() { - let ctx = create_time_dimension_context(); - let err = compile_pre_agg(&ctx, "broken_rollup_unsupported") - .expect_err("Pre-aggregation with unresolvable time dimension should fail to compile"); + fn test_interpolated_references_resolve_to_member_symbols() { + let ctx = create_reference_context(); + let compiled = compile_pre_agg(&ctx, "interpolated_rollup").unwrap(); + assert_eq!( - err.message, - "'created_at' not found for path 'orders.created_at' in pre-aggregation 'orders.broken_rollup_unsupported'" + compiled + .measures + .iter() + .map(|m| m.full_name()) + .collect_vec(), + vec!["orders.count".to_string()] + ); + assert_eq!( + compiled + .dimensions + .iter() + .map(|d| d.full_name()) + .collect_vec(), + vec!["orders.status".to_string()] + ); + assert_eq!( + compiled + .segments + .iter() + .map(|sg| sg.full_name()) + .collect_vec(), + vec!["expr:orders.completed".to_string()] + ); + assert_eq!(compiled.time_dimensions.len(), 1); + assert_eq!( + compiled.time_dimensions[0].full_name(), + "orders.created_at_day" ); + assert_eq!(compiled.granularity, Some("day".to_string())); } #[test] - fn test_time_dimension_resolved_to_cube_ref_without_granularity_returns_error() { - let ctx = create_time_dimension_context(); - let err = compile_pre_agg(&ctx, "broken_rollup_no_granularity") - .expect_err("Pre-aggregation with unresolvable time dimension should fail to compile"); + fn test_interpolated_time_dimension_without_granularity_resolves() { + let ctx = create_reference_context(); + let compiled = compile_pre_agg(&ctx, "interpolated_rollup_no_granularity").unwrap(); + + assert_eq!(compiled.time_dimensions.len(), 1); + assert_eq!(compiled.time_dimensions[0].full_name(), "orders.created_at"); + assert_eq!(compiled.granularity, None); + } + + // One list mixing both forms keeps every member it names, in declaration + // order — join hints and lambda member matching read the list positionally. + #[test] + fn test_interpolated_and_symbol_references_in_one_list() { + let ctx = create_reference_context(); + let compiled = compile_pre_agg(&ctx, "interpolated_rollup_mixed_list").unwrap(); + assert_eq!( - err.message, - "'created_at' not found for path 'orders.created_at' in pre-aggregation 'orders.broken_rollup_no_granularity'" + compiled + .measures + .iter() + .map(|m| m.full_name()) + .collect_vec(), + vec!["orders.total".to_string(), "orders.count".to_string()] + ); + assert_eq!( + compiled + .dimensions + .iter() + .map(|d| d.full_name()) + .collect_vec(), + vec!["orders.status".to_string(), "orders.city".to_string()] + ); + assert_eq!( + compiled.time_dimensions[0].full_name(), + "orders.created_at_day" ); } - // Interpolated reference with a granularity-suffixed member name, - // e.g. `(CUBE) => `${CUBE}.created_at_day``. + // An interpolated reference naming the granularity-suffixed member instead + // of the member itself, e.g. `(CUBE) => `${CUBE}.created_at_day``. #[test] - fn test_time_dimension_with_granularity_suffix_returns_error() { - let ctx = create_time_dimension_context(); + fn test_interpolated_time_dimension_with_granularity_suffix_returns_error() { + let ctx = create_reference_context(); let err = compile_pre_agg(&ctx, "broken_rollup_granularity_suffix") .expect_err("Pre-aggregation with unresolvable time dimension should fail to compile"); assert_eq!( @@ -750,6 +901,48 @@ mod tests { ); } + // Naming the granularity inside the reference, `${CUBE}.created_at.day`, + // instead of through `granularity:`. Rejected — and rejected the same way as + // the equivalent symbol reference `CUBE.created_at.day`. + #[test] + fn test_granularity_inside_the_reference_is_rejected_like_the_symbol_form() { + let ctx = create_reference_context(); + let interpolated = compile_pre_agg(&ctx, "interpolated_rollup_granularity_segment") + .expect_err("Granularity inside a time dimension reference should fail to compile"); + let symbol = compile_pre_agg(&ctx, "symbol_rollup_granularity_segment") + .expect_err("Granularity inside a time dimension reference should fail to compile"); + + assert_eq!(interpolated.message, symbol.message); + assert_eq!( + interpolated.message, + "Pre-aggregation time dimension must be a dimension" + ); + } + + #[test] + fn test_interpolated_measure_that_does_not_exist_returns_error() { + let ctx = create_reference_context(); + let err = compile_pre_agg(&ctx, "broken_rollup_unknown_measure") + .expect_err("Pre-aggregation with unresolvable measure should fail to compile"); + assert_eq!( + err.message, + "'unknown_total' not found for path 'orders.unknown_total' in pre-aggregation 'orders.broken_rollup_unknown_measure'" + ); + } + + // An element built as an expression rather than a member reference names no + // member, so it is reported instead of dropped from the reference list. + #[test] + fn test_reference_that_names_no_member_returns_error() { + let ctx = create_reference_context(); + let err = compile_pre_agg(&ctx, "broken_rollup_expression_dimension") + .expect_err("Pre-aggregation with an expression reference should fail to compile"); + assert_eq!( + err.message, + "'status = 'completed'' not found for path 'orders.status = 'completed'' in pre-aggregation 'orders.broken_rollup_expression_dimension'" + ); + } + #[test] fn test_compile_simple_rollup() { let schema = MockSchema::from_yaml_file("common/pre_aggregations_test.yaml"); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_call.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_call.rs index b7aee3eb3417f..af0478eca5500 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_call.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_call.rs @@ -9,9 +9,18 @@ use crate::planner::{CubeNameSymbol, CubeTableSymbol}; use crate::utils::sql_expression_scanner::analyze_template_arg_contexts; use cubenativeutils::CubeError; use itertools::Itertools; +use lazy_static::lazy_static; +use regex::Regex; use std::collections::HashMap; use std::rc::Rc; +lazy_static! { + /// A whole template element made of a cube-name placeholder followed only by + /// dotted identifiers. + static ref INTERPOLATED_REFERENCE_RE: Regex = + Regex::new(r"^\s*\{arg:(\d+)\}((?:\.[A-Za-z_$][A-Za-z0-9_$]*)+)\s*$").unwrap(); +} + /// Reference to a cube from a SQL template. /// /// - `Name` — the cube as an identifier (rendered from `{CUBE}` or @@ -88,6 +97,18 @@ impl SqlDependency { } } +/// What one element of a reference declaration names, as read off the compiled +/// template by `SqlCall::reference_items`. +#[derive(Clone, Debug)] +pub enum SqlCallReference { + Symbol(Rc), + /// Member path of an element that interpolated the cube name, still to be + /// resolved against the data model. + Path(Vec), + /// Element naming no member, rendered for diagnostics. + Unresolved(String), +} + /// Namespace for the placeholder prefixes recognised inside a /// `SqlCall` template: /// @@ -547,6 +568,117 @@ impl SqlCall { } } + /// What each element of a reference declaration (a pre-aggregation + /// `measures:` / `dimensions:` / `segments:` / `time_dimension:`) names, + /// in declaration order. + /// + /// An element referencing a member yields that member's symbol. An element + /// that interpolated the cube itself — ``(CUBE) => `${CUBE}.created_at` `` — + /// depends on the cube name and keeps the member as literal text, so it + /// yields the path to resolve: the cube reference's path followed by the + /// literal segments. Anything else names no member and is reported as + /// unresolved, rendered for diagnostics. + /// + /// An element wrapping a member reference in an expression still yields that + /// member, since the member is a dependency of its own — only the cube-name + /// form has nothing to fall back on. + pub fn reference_items(&self) -> Vec { + let elements = match &self.template { + SqlTemplate::String(s) => std::slice::from_ref(s), + SqlTemplate::StringVec(strings) => strings.as_slice(), + }; + let mut taken = vec![false; self.deps.len()]; + let mut result = Vec::new(); + for element in elements { + let arg_indices = Self::template_arg_indices(element); + let names_symbol = arg_indices + .iter() + .any(|index| self.deps.get(*index).is_some_and(|dep| dep.is_symbol())); + if names_symbol { + for index in arg_indices { + // An index the recorded dependencies don't cover names + // nothing; the rest of the element is still read. + let Some(symbol) = self.deps.get(index).and_then(|dep| dep.as_symbol()) else { + continue; + }; + if taken[index] { + continue; + } + taken[index] = true; + result.push(SqlCallReference::Symbol(symbol.clone())); + } + continue; + } + match self.interpolated_reference_path(element) { + Some(path) => result.push(SqlCallReference::Path(path)), + None => result.push(SqlCallReference::Unresolved( + self.render_for_diagnostics(element), + )), + } + } + // A symbol no element referenced: keep it rather than lose a member the + // declaration depends on. + for (index, dep) in self.deps.iter().enumerate() { + if !taken[index] { + if let Some(symbol) = dep.as_symbol() { + result.push(SqlCallReference::Symbol(symbol.clone())); + } + } + } + result + } + + // Path of an element made of a cube-name placeholder followed only by dotted + // identifiers; `None` when the element has any other shape. + fn interpolated_reference_path(&self, element: &str) -> Option> { + let captures = INTERPOLATED_REFERENCE_RE.captures(element)?; + let index = captures.get(1)?.as_str().parse::().ok()?; + let cube_ref = self.deps.get(index)?.as_cube_ref()?.as_name()?; + let mut path = cube_ref.path().clone(); + path.extend( + captures + .get(2)? + .as_str() + .split('.') + .skip(1) + .map(String::from), + ); + Some(path) + } + + // `{arg:N}` indices in the order they appear in the element. + fn template_arg_indices(element: &str) -> Vec { + let needle = format!("{{{}:", SqlCallArg::ARG_PREFIX); + let mut result = Vec::new(); + let mut rest = element; + while let Some(start) = rest.find(&needle) { + rest = &rest[start + needle.len()..]; + let Some(end) = rest.find('}') else { + break; + }; + if let Ok(index) = rest[..end].parse::() { + result.push(index); + } + rest = &rest[end + 1..]; + } + result + } + + // Element with its dependencies replaced by the members and cubes they name, + // for error messages about an element that names no member. + fn render_for_diagnostics(&self, element: &str) -> String { + let deps = self + .deps + .iter() + .map(|dep| match dep { + SqlDependency::Symbol(symbol) => symbol.full_name(), + SqlDependency::CubeRef(cube_ref) => cube_ref.cube_name().clone(), + }) + .collect_vec(); + Self::substitute_template(element, &deps, &[], &[], &[], &[]) + .unwrap_or_else(|_| element.to_string()) + } + /// Number of member-symbol dependencies. Cube refs are not /// counted. pub fn dependencies_count(&self) -> usize { @@ -703,3 +835,206 @@ impl crate::utils::debug::DebugSql for SqlCall { .unwrap() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_fixtures::cube_bridge::MockSchema; + use crate::test_fixtures::test_utils::TestContext; + use indoc::indoc; + + fn test_context() -> TestContext { + let schema = MockSchema::from_yaml(indoc! {" + cubes: + - name: orders + sql: SELECT * FROM orders + dimensions: + - name: id + type: number + sql: id + primary_key: true + - name: status + type: string + sql: status + - name: created_at + type: time + sql: created_at + measures: + - name: count + type: count + "}) + .unwrap(); + TestContext::new(schema).unwrap() + } + + fn cube_name_dep(cube_name: &str, path: &[&str]) -> SqlDependency { + SqlDependency::CubeRef(CubeRef::Name(CubeNameSymbol::new( + cube_name.to_string(), + path.iter().map(|p| p.to_string()).collect(), + ))) + } + + fn call(template: SqlTemplate, deps: Vec) -> SqlCall { + SqlCall::new( + template, + deps, + vec![], + vec![], + SecutityContextProps::default(), + ) + } + + fn single(template: &str, deps: Vec) -> SqlCall { + call(SqlTemplate::String(template.to_string()), deps) + } + + /// Reference items as comparable strings: `symbol:`, + /// `path:`, `unresolved:`. + fn described(sql_call: &SqlCall) -> Vec { + sql_call + .reference_items() + .iter() + .map(|item| match item { + SqlCallReference::Symbol(symbol) => format!("symbol:{}", symbol.full_name()), + SqlCallReference::Path(path) => format!("path:{}", path.join(".")), + SqlCallReference::Unresolved(rendered) => format!("unresolved:{}", rendered), + }) + .collect() + } + + #[test] + fn test_cube_name_followed_by_member_yields_path() { + let sql_call = single("{arg:0}.created_at", vec![cube_name_dep("orders", &[])]); + + assert_eq!(described(&sql_call), vec!["path:orders.created_at"]); + } + + #[test] + fn test_surrounding_whitespace_is_ignored() { + let sql_call = single(" {arg:0}.created_at\n", vec![cube_name_dep("orders", &[])]); + + assert_eq!(described(&sql_call), vec!["path:orders.created_at"]); + } + + #[test] + fn test_join_path_is_kept() { + // `${CUBE.users}.name` — the cube reference carries the cubes traversed, + // and the member name follows as literal text. + let sql_call = single("{arg:0}.name", vec![cube_name_dep("users", &["orders"])]); + + assert_eq!(described(&sql_call), vec!["path:orders.users.name"]); + } + + #[test] + fn test_literal_segments_after_the_cube_are_all_kept() { + let sql_call = single("{arg:0}.users.name", vec![cube_name_dep("orders", &[])]); + + assert_eq!(described(&sql_call), vec!["path:orders.users.name"]); + } + + #[test] + fn test_every_element_of_a_reference_list_is_resolved() { + let sql_call = call( + SqlTemplate::StringVec(vec![ + "{arg:0}.count".to_string(), + "{arg:1}.name".to_string(), + ]), + vec![cube_name_dep("orders", &[]), cube_name_dep("users", &[])], + ); + + assert_eq!( + described(&sql_call), + vec!["path:orders.count", "path:users.name"] + ); + } + + // A member reference produces a symbol dependency of its own. + #[test] + fn test_member_dependency_is_reported_as_a_symbol() { + let ctx = test_context(); + let symbol = ctx.create_dimension("orders.created_at").unwrap(); + let sql_call = single("{arg:0}", vec![SqlDependency::Symbol(symbol)]); + + assert_eq!(described(&sql_call), vec!["symbol:orders.created_at"]); + } + + // Declaration order survives a list mixing both forms — join hints and + // lambda member matching read the compiled list positionally. + #[test] + fn test_declaration_order_is_kept_for_a_mixed_list() { + let ctx = test_context(); + let symbol = ctx.create_dimension("orders.status").unwrap(); + let sql_call = call( + SqlTemplate::StringVec(vec![ + "{arg:0}.created_at".to_string(), + "{arg:1}".to_string(), + ]), + vec![cube_name_dep("orders", &[]), SqlDependency::Symbol(symbol)], + ); + + assert_eq!( + described(&sql_call), + vec!["path:orders.created_at", "symbol:orders.status"] + ); + } + + // An element naming a member through an expression names no single member. + #[test] + fn test_element_carrying_an_expression_is_unresolved() { + for (template, expected) in [ + ("{arg:0}.created_at + 1", "unresolved:orders.created_at + 1"), + ( + "date_trunc('day', {arg:0}.created_at)", + "unresolved:date_trunc('day', orders.created_at)", + ), + ("{arg:0}", "unresolved:orders"), + ("{arg:0}.", "unresolved:orders."), + ("{arg:0}.2days", "unresolved:orders.2days"), + ] { + let sql_call = single(template, vec![cube_name_dep("orders", &[])]); + + assert_eq!( + described(&sql_call), + vec![expected], + "unexpected reference items for `{}`", + template + ); + } + } + + // `${CUBE.sql()}` renders the cube's table expression, not its name, so it + // cannot start a member path. + #[test] + fn test_cube_table_reference_is_unresolved() { + let ctx = test_context(); + let cube_table = ctx + .query_tools() + .compiler() + .borrow_mut() + .add_cube_table_evaluator("orders".to_string(), vec![]) + .unwrap(); + let sql_call = single( + "{arg:0}.created_at", + vec![SqlDependency::CubeRef(CubeRef::Table(cube_table))], + ); + + assert_eq!(described(&sql_call), vec!["unresolved:orders.created_at"]); + } + + // An index the recorded dependencies don't cover must not be read as one. + #[test] + fn test_placeholder_out_of_bounds_next_to_a_member_is_skipped() { + let ctx = test_context(); + let symbol = ctx.create_dimension("orders.status").unwrap(); + let sql_call = single("{arg:0} || {arg:7}", vec![SqlDependency::Symbol(symbol)]); + + assert_eq!(described(&sql_call), vec!["symbol:orders.status"]); + } + + #[test] + fn test_placeholder_out_of_bounds_is_unresolved() { + let sql_call = single("{arg:3}.created_at", vec![cube_name_dep("orders", &[])]); + + assert_eq!(described(&sql_call), vec!["unresolved:{arg:3}.created_at"]); + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_member_sql.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_member_sql.rs index 0e9546017538a..f7c734fe72255 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_member_sql.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_member_sql.rs @@ -104,12 +104,63 @@ impl MockMemberSql { })) } + /// Pre-aggregation array references where an element may interpolate the + /// cube itself: `["{CUBE}.count", "{CUBE.status}", "city"]`. A brace-free + /// element is a plain member path, recorded the way + /// `pre_agg_array_refs` records it, so both forms can be mixed in one list. + pub fn pre_agg_array_templates(members: Vec) -> Result, CubeError> { + let mut args = SqlTemplateArgs::default(); + let mut args_names = Vec::new(); + let mut template_elements = Vec::new(); + + for member in &members { + if member.contains('{') { + template_elements.push(Self::parse_template_into( + member, + &mut args, + &mut args_names, + )?); + } else { + let path_parts: Vec = member.split('.').map(|s| s.to_string()).collect(); + if path_parts.iter().any(|p| p.is_empty()) { + return Err(CubeError::user(format!( + "Invalid path in pre-aggregation: {}", + member + ))); + } + let arg_name = path_parts[0].clone(); + if !args_names.contains(&arg_name) { + args_names.push(arg_name); + } + let index = args.insert_symbol_path(path_parts); + template_elements.push(format!("{{arg:{}}}", index)); + } + } + + Ok(Rc::new(Self { + template: SqlTemplate::StringVec(template_elements), + args, + args_names, + })) + } + /// Parse the template string and extract symbol paths /// Converts "{path.to.symbol}" to "{arg:N}" and collects paths fn parse_template(template: &str) -> Result<(String, SqlTemplateArgs, Vec), CubeError> { - let mut result = String::new(); let mut args = SqlTemplateArgs::default(); let mut args_names = Vec::new(); + let result = Self::parse_template_into(template, &mut args, &mut args_names)?; + Ok((result, args, args_names)) + } + + // Parses one template, recording its dependencies into the given args so + // several templates can share one dependency list. + fn parse_template_into( + template: &str, + args: &mut SqlTemplateArgs, + args_names: &mut Vec, + ) -> Result { + let mut result = String::new(); let mut chars = template.chars().peekable(); @@ -172,8 +223,7 @@ impl MockMemberSql { // planner passes at render time. if let Some(body) = path.strip_prefix("FILTER_PARAMS:") { let (cube_name, name, column) = Self::parse_filter_params_body(body)?; - let column = - Self::parse_column_references(&column, &mut args, &mut args_names)?; + let column = Self::parse_column_references(&column, args, args_names)?; let index = args.insert_filter_params(FilterParamsItem { cube_name, name, @@ -216,7 +266,7 @@ impl MockMemberSql { } } - Ok((result, args, args_names)) + Ok(result) } // Splits a `.:` FILTER_PARAMS body. diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/pre_aggregation.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/pre_aggregation.rs index 4a7323e46b0e3..4e6f93e90f1b7 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/pre_aggregation.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/yaml/pre_aggregation.rs @@ -161,7 +161,11 @@ impl YamlPreAggregationDefinition { } fn build_array_references(members: Vec) -> Result, CubeError> { - MockMemberSql::pre_agg_array_refs(members).map(|m| m as Rc) + if members.iter().any(|m| m.contains('{')) { + MockMemberSql::pre_agg_array_templates(members).map(|m| m as Rc) + } else { + MockMemberSql::pre_agg_array_refs(members).map(|m| m as Rc) + } } fn build_single_reference(member: String) -> Result, CubeError> { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/pre_aggregation_matching_test.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/pre_aggregation_matching_test.yaml index a5f9ea15659f9..c2c4b569915f4 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/pre_aggregation_matching_test.yaml +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/pre_aggregation_matching_test.yaml @@ -181,3 +181,17 @@ cubes: - id - status - city + + # Same members as `segment_rollup`, but every reference is written as a + # cube-name interpolation — `(CUBE) => `${CUBE}.count``. + - name: interpolated_refs_rollup + type: rollup + measures: + - '{CUBE}.count' + - '{CUBE}.total_amount' + dimensions: + - '{CUBE}.status' + segments: + - '{CUBE}.high_priority' + time_dimension: '{CUBE}.created_at' + granularity: day diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/snapshots/cubesqlplanner__tests__integration__pre_aggregations__sql_generation__interpolated_refs_full_match_result.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/snapshots/cubesqlplanner__tests__integration__pre_aggregations__sql_generation__interpolated_refs_full_match_result.snap new file mode 100644 index 0000000000000..b0e7963d1d1d1 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/snapshots/cubesqlplanner__tests__integration__pre_aggregations__sql_generation__interpolated_refs_full_match_result.snap @@ -0,0 +1,11 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/sql_generation.rs +expression: result +--- +orders__status | orders__created_at_day | orders__count | orders__total_amount +---------------+------------------------+---------------+--------------------- +cancelled | 2025-02-15 00:00:00 | 1 | 25.00 +completed | 2025-01-10 00:00:00 | 1 | 100.00 +completed | 2025-01-31 00:00:00 | 1 | 300.00 +pending | 2025-01-10 00:00:00 | 1 | 200.00 +pending | 2025-03-01 00:00:00 | 1 | 175.00 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/snapshots/cubesqlplanner__tests__integration__pre_aggregations__sql_generation__interpolated_refs_with_coarser_granularity_result.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/snapshots/cubesqlplanner__tests__integration__pre_aggregations__sql_generation__interpolated_refs_with_coarser_granularity_result.snap new file mode 100644 index 0000000000000..a24338aecc9f1 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/snapshots/cubesqlplanner__tests__integration__pre_aggregations__sql_generation__interpolated_refs_with_coarser_granularity_result.snap @@ -0,0 +1,10 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/sql_generation.rs +expression: result +--- +orders__status | orders__created_at_month | orders__count +---------------+--------------------------+-------------- +cancelled | 2025-02-01 00:00:00 | 1 +completed | 2025-01-01 00:00:00 | 2 +pending | 2025-01-01 00:00:00 | 1 +pending | 2025-03-01 00:00:00 | 1 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/sql_generation.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/sql_generation.rs index 8994412eda9a1..54faf53106fa5 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/sql_generation.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/pre_aggregations/sql_generation.rs @@ -1692,3 +1692,101 @@ async fn test_ungrouped_cross_cube_view_query_matches_rollup_covering_both_prima Ok(()) } + +// --- References written as a cube-name interpolation --- +// +// `interpolated_refs_rollup` names the same members as `segment_rollup`, but +// through `` `${CUBE}.count` ``-style references. The two must be picked for the +// same queries and read back the same rows. The rows themselves are pinned by +// the snapshot, which is only compared when Postgres execution is enabled. + +async fn interpolated_and_symbol_refs_agree( + query_yaml: &str, + snapshot_name: &str, +) -> Result<(), CubeError> { + let interpolated_ctx = TestContext::new( + MockSchema::from_yaml_file("common/pre_aggregation_matching_test.yaml") + .only_pre_aggregations(&["interpolated_refs_rollup"]), + )?; + let symbol_ctx = TestContext::new( + MockSchema::from_yaml_file("common/pre_aggregation_matching_test.yaml") + .only_pre_aggregations(&["segment_rollup"]), + )?; + + let (_sql, interpolated_pre_aggrs) = + interpolated_ctx.build_sql_with_used_pre_aggregations(query_yaml)?; + assert_eq!(interpolated_pre_aggrs.len(), 1); + assert_eq!(interpolated_pre_aggrs[0].name(), "interpolated_refs_rollup"); + + let (_sql, symbol_pre_aggrs) = symbol_ctx.build_sql_with_used_pre_aggregations(query_yaml)?; + assert_eq!(symbol_pre_aggrs.len(), 1); + assert_eq!(symbol_pre_aggrs[0].name(), "segment_rollup"); + + let interpolated_result = interpolated_ctx + .try_execute_pg(query_yaml, "pre_aggregation_matching_tables.sql") + .await; + let symbol_result = symbol_ctx + .try_execute_pg(query_yaml, "pre_aggregation_matching_tables.sql") + .await; + + assert_eq!(interpolated_result, symbol_result); + + // Without Postgres execution there are no rows to compare, so make it + // explicit that the row assertions below do run when it is enabled. + #[cfg(feature = "integration-postgres")] + assert!( + interpolated_result.is_some(), + "Postgres execution is enabled but returned no result" + ); + + if let Some(result) = interpolated_result { + insta::assert_snapshot!(snapshot_name, result); + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_interpolated_refs_full_match() -> Result<(), CubeError> { + interpolated_and_symbol_refs_agree( + indoc! {" + measures: + - orders.count + - orders.total_amount + dimensions: + - orders.status + segments: + - orders.high_priority + time_dimensions: + - dimension: orders.created_at + granularity: day + order: + - id: orders.status + - id: orders.created_at + "}, + "interpolated_refs_full_match_result", + ) + .await +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_interpolated_refs_with_coarser_granularity() -> Result<(), CubeError> { + interpolated_and_symbol_refs_agree( + indoc! {" + measures: + - orders.count + dimensions: + - orders.status + segments: + - orders.high_priority + time_dimensions: + - dimension: orders.created_at + granularity: month + order: + - id: orders.status + - id: orders.created_at + "}, + "interpolated_refs_with_coarser_granularity_result", + ) + .await +}