diff --git a/docs-mintlify/admin/deployment/oidc/aws.mdx b/docs-mintlify/admin/deployment/oidc/aws.mdx index 23409ecf1b05d..141dd82a330c0 100644 --- a/docs-mintlify/admin/deployment/oidc/aws.mdx +++ b/docs-mintlify/admin/deployment/oidc/aws.mdx @@ -309,6 +309,21 @@ deployment's default identity is the simplest place to put this. + + +OIDC only covers Cube's **read** side of the export bucket. The data +warehouse itself (Snowflake, Redshift, Athena, BigQuery, …) runs the +`UNLOAD` that writes objects to the bucket, and the warehouse cannot +federate with Cube's OIDC issuer. You still need to provide **separate +credentials for the `UNLOAD`** so the warehouse can write to S3 — typically +an AWS access key pair or a warehouse-side storage integration / IAM role +— via the standard export bucket env vars (e.g. +`CUBEJS_DB_EXPORT_BUCKET_AWS_KEY` and `CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET`, +or the driver-specific storage-integration variables). OIDC then handles +Cube's download of the unloaded objects from the bucket. + + + ## Cube Store CSPS bucket Cube Store CSPS lets you store pre-aggregations in your own S3 bucket. diff --git a/docs-mintlify/admin/deployment/oidc/azure.mdx b/docs-mintlify/admin/deployment/oidc/azure.mdx index a199b9cbb38e2..0bae480f93e92 100644 --- a/docs-mintlify/admin/deployment/oidc/azure.mdx +++ b/docs-mintlify/admin/deployment/oidc/azure.mdx @@ -231,6 +231,21 @@ Contributor** on the storage account. + + +OIDC only covers Cube's **read** side of the export bucket. The data +warehouse itself (Snowflake on Azure, Synapse, …) runs the `UNLOAD` that +writes objects to Blob Storage, and the warehouse cannot federate with +Cube's OIDC issuer. You still need to provide **separate credentials for +the unload** so the warehouse can write to the container — typically a +storage account key, SAS token, or a warehouse-side storage integration — +via the standard export bucket env vars (e.g. +`CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY`, or the driver-specific +storage-integration variables). OIDC then handles Cube's download of the +unloaded objects from the bucket. + + + ## Scaling past 20 federated credentials A single app registration accepts at most **20 federated credentials**. diff --git a/docs-mintlify/admin/deployment/oidc/gcp.mdx b/docs-mintlify/admin/deployment/oidc/gcp.mdx index 788543b6509d0..0e3e1cd35c4dc 100644 --- a/docs-mintlify/admin/deployment/oidc/gcp.mdx +++ b/docs-mintlify/admin/deployment/oidc/gcp.mdx @@ -261,6 +261,21 @@ deployment's service account read / write access to the bucket. + + +OIDC only covers Cube's **read** side of the export bucket. The data +warehouse itself (BigQuery, Snowflake on GCP, …) runs the `UNLOAD` / +`EXPORT DATA` that writes objects to the bucket, and the warehouse cannot +federate with Cube's OIDC issuer. You still need to provide **separate +credentials for the unload** so the warehouse can write to GCS — typically +an HMAC key pair or a warehouse-side service-account integration — via the +standard export bucket env vars (e.g. +`CUBEJS_DB_EXPORT_GCS_CREDENTIALS`, or the driver-specific +storage-integration variables). OIDC then handles Cube's download of the +unloaded objects from the bucket. + + + ## Direct federation If you'd rather skip the service account impersonation hop, grant diff --git a/docs-mintlify/admin/deployment/oidc/index.mdx b/docs-mintlify/admin/deployment/oidc/index.mdx index effa1add7b694..128750adf38f7 100644 --- a/docs-mintlify/admin/deployment/oidc/index.mdx +++ b/docs-mintlify/admin/deployment/oidc/index.mdx @@ -24,7 +24,9 @@ You can use OIDC workload identity to authenticate to: - **Data sources** — AWS Athena, Redshift, BigQuery, Snowflake, and any other driver that supports federated credentials. - **Export buckets** — S3 and GCS buckets used for `EXPORT_BUCKET` pre-aggregation - unloads. + unloads. OIDC covers Cube's download of the unloaded objects; the warehouse's + `UNLOAD` write still needs its own credentials configured on the client side + — see the per-cloud guides for details. - **Cube Store CSPS** — a per-deployment S3 / GCS bucket that holds your Cube Store pre-aggregations (Customer-Supplied Pre-aggregation Storage). - **Bring-your-own LLM providers** — AWS Bedrock, Google Vertex AI, and Azure diff --git a/packages/cubejs-backend-native/src/orchestrator.rs b/packages/cubejs-backend-native/src/orchestrator.rs index 521577f697647..9b41660b2b606 100644 --- a/packages/cubejs-backend-native/src/orchestrator.rs +++ b/packages/cubejs-backend-native/src/orchestrator.rs @@ -11,8 +11,8 @@ use neon::context::{Context, FunctionContext, ModuleContext}; use neon::handle::Handle; use neon::object::Object; use neon::prelude::{ - JsArray, JsArrayBuffer, JsBox, JsBuffer, JsFunction, JsObject, JsPromise, JsResult, JsValue, - NeonResult, + JsArray, JsArrayBuffer, JsBox, JsBuffer, JsFunction, JsObject, JsPromise, JsResult, JsString, + JsValue, NeonResult, }; use neon::types::buffer::TypedArray; use serde::Deserialize; @@ -330,21 +330,24 @@ pub fn get_cubestore_result(mut cx: FunctionContext) -> JsResult { let result = cx.argument::>>(0)?; let js_array = cx.execute_scoped(|mut cx| { + let js_keys: Vec> = result.members.iter().map(|k| cx.string(k)).collect(); + let js_array = JsArray::new(&mut cx, result.rows.len()); for (i, row) in result.rows.iter().enumerate() { let js_row = cx.execute_scoped(|mut cx| { let js_row = JsObject::new(&mut cx); - for (key, value) in result.members.iter().zip(row.iter()) { - let js_key = cx.string(key); + + for (js_key, value) in js_keys.iter().zip(row.iter()) { let js_value: Handle<'_, JsValue> = match value { DBResponsePrimitive::Null => cx.null().upcast(), // For compatibility, we convert all primitives to strings other => cx.string(other.to_string()).upcast(), }; - js_row.set(&mut cx, js_key, js_value)?; + js_row.set(&mut cx, *js_key, js_value)?; } + Ok(js_row) })?; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs deleted file mode 100644 index 10c4e8f0880dc..0000000000000 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs +++ /dev/null @@ -1,55 +0,0 @@ -use crate::logical_plan::pretty_print::*; -use crate::planner::planners::multi_stage::MultiStageAppliedState; - -impl PrettyPrint for MultiStageAppliedState { - fn pretty_print(&self, result: &mut PrettyPrintResult, state: &PrettyPrintState) { - let details_state = state.new_level(); - result.println( - &format!( - "-time_dimensions: {}", - print_symbols(&self.time_dimensions()) - ), - state, - ); - - result.println( - &format!("-dimensions: {}", print_symbols(&self.dimensions())), - state, - ); - - result.println("dimensions_filters:", &state); - for filter in self.dimensions_filters().iter() { - pretty_print_filter_item(result, &details_state, filter); - } - result.println("time_dimensions_filters:", &state); - for filter in self.time_dimensions_filters().iter() { - pretty_print_filter_item(result, &details_state, filter); - } - result.println("measures_filter:", &state); - for filter in self.measures_filters().iter() { - pretty_print_filter_item(result, &details_state, filter); - } - result.println("segments:", &state); - for filter in self.segments().iter() { - pretty_print_filter_item(result, &details_state, filter); - } - - result.println("time_shifts:", &state); - for (_, time_shift) in self.time_shifts().dimensions_shifts.iter() { - result.println( - &format!( - "- {}: {}", - time_shift.dimension.full_name(), - if let Some(interval) = &time_shift.interval { - interval.to_sql() - } else if let Some(name) = &time_shift.name { - format!("{} (named)", name.to_string()) - } else { - "None".to_string() - } - ), - &details_state, - ); - } - } -} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/mod.rs index 3a82b775bc971..3d269880839f5 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/mod.rs @@ -1,5 +1,4 @@ mod calculation; -mod common; mod dimension; mod get_date_range; mod leaf_measure; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index f64d382d9f3e7..9c1b11ad3cc34 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -1,6 +1,6 @@ use super::query_tools::QueryTools; use super::top_level_planner::TopLevelPlanner; -use super::QueryProperties; +use super::{QueryProperties, QueryPropertiesCompiler}; use crate::cube_bridge::base_query_options::BaseQueryOptions; use crate::cube_bridge::pre_aggregation_obj::NativePreAggregationObj; use crate::logical_plan::PreAggregationUsage; @@ -61,7 +61,7 @@ impl BaseQuery { options.static_data().member_to_alias.clone(), )?; - let request = QueryProperties::try_new(query_tools.clone(), options)?; + let request = QueryPropertiesCompiler::new(query_tools.clone()).build(options)?; Ok(Self { context, diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/mod.rs index 34904dc3403f0..c0db55ab240ff 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/mod.rs @@ -15,6 +15,7 @@ pub mod visitor; pub mod params_allocator; pub mod planners; pub mod query_properties; +pub mod query_properties_compiler; pub mod query_tools; pub mod sql_templates; pub mod top_level_planner; @@ -29,6 +30,7 @@ pub use compiler::Compiler; pub use join_hints::JoinHints; pub use params_allocator::ParamsAllocator; pub use query_properties::{FullKeyAggregateMeasures, OrderByItem, QueryProperties}; +pub use query_properties_compiler::QueryPropertiesCompiler; pub use sql_call::*; pub use symbols::*; pub use time_dimension::*; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs index 8a669c4ce1e53..b00908fa23313 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs @@ -59,13 +59,6 @@ pub struct MeasuresJoinHints { } impl MeasuresJoinHints { - pub fn empty() -> Self { - Self { - base_hints: JoinHints::new(), - measure_hints: vec![], - } - } - pub fn builder(query_join_hints: &JoinHints) -> MeasuresJoinHintsBuilder { MeasuresJoinHintsBuilder { initial_hints: query_join_hints.clone(), @@ -137,16 +130,6 @@ pub struct MultiFactJoinGroups { } impl MultiFactJoinGroups { - pub fn empty(query_tools: Rc) -> Self { - Self { - query_tools, - measures_join_hints: MeasuresJoinHints::empty(), - groups: vec![], - dimension_paths: HashMap::new(), - measure_paths: HashMap::new(), - } - } - pub fn try_new( query_tools: Rc, measures_join_hints: MeasuresJoinHints, diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs index b123cd1146bea..64dda0181cd8a 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/dimension_subquery_planner.rs @@ -3,7 +3,6 @@ use crate::logical_plan::{pretty_print_rc, DimensionSubQuery}; use crate::physical_plan::QualifiedColumnName; use crate::planner::collectors::collect_sub_query_dimensions; use crate::planner::filter::FilterItem; -use crate::planner::join_hints::JoinHints; use crate::planner::query_tools::QueryTools; use crate::planner::QueryProperties; use crate::planner::{MemberExpressionExpression, MemberExpressionSymbol, MemberSymbol}; @@ -111,26 +110,17 @@ impl DimensionSubqueryPlanner { (vec![], vec![]) }; - let sub_query_properties = QueryProperties::try_new_from_precompiled( - self.query_tools.clone(), - vec![measure.clone()], //measures, - primary_keys_dimensions.clone(), - vec![], - time_dimensions_filters, - dimensions_filters, - vec![], - vec![], - vec![], - None, - None, - true, - false, - false, - false, - Rc::new(JoinHints::new()), - true, - self.query_properties.disable_external_pre_aggregations(), - )?; + let sub_query_properties = QueryProperties::builder() + .query_tools(self.query_tools.clone()) + .measures(vec![measure.clone()]) + .dimensions(primary_keys_dimensions.clone()) + .time_dimensions_filters(time_dimensions_filters) + .dimensions_filters(dimensions_filters) + .ignore_cumulative(true) + .disable_external_pre_aggregations( + self.query_properties.disable_external_pre_aggregations(), + ) + .build()?; let query_planner = QueryPlanner::new(sub_query_properties, self.query_tools.clone()); let sub_query = query_planner.plan()?; let result = Rc::new(DimensionSubQuery { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs index 25e0a862acf2b..6fbf06027f756 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/full_key_query_aggregate_planner.rs @@ -63,7 +63,7 @@ impl FullKeyAggregateQueryPlanner { offset: self.query_properties.offset(), limit: self.query_properties.row_limit(), ungrouped: self.query_properties.ungrouped(), - order_by: self.query_properties.order_by().clone(), + order_by: self.query_properties.order_by().to_vec(), })) .source(source) .build(); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs deleted file mode 100644 index 86d56d24330dc..0000000000000 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs +++ /dev/null @@ -1,547 +0,0 @@ -use crate::planner::collectors::has_multi_stage_members; -use crate::planner::filter::FilterOperator; -use crate::planner::filter::{FilterGroup, FilterItem}; -use crate::planner::planners::multi_stage::time_shift_state::TimeShiftState; -use crate::planner::{DimensionTimeShift, MeasureTimeShifts, MemberSymbol}; -use cubenativeutils::CubeError; -use itertools::Itertools; -use std::cmp::PartialEq; -use std::collections::HashSet; -use std::fmt::Debug; -use std::rc::Rc; - -#[derive(Clone)] -pub struct MultiStageAppliedState { - time_dimensions: Vec>, - dimensions: Vec>, - time_dimensions_filters: Vec, - dimensions_filters: Vec, - measures_filters: Vec, - segments: Vec, - time_shifts: TimeShiftState, -} - -impl MultiStageAppliedState { - pub fn new( - time_dimensions: Vec>, - dimensions: Vec>, - time_dimensions_filters: Vec, - dimensions_filters: Vec, - measures_filters: Vec, - segments: Vec, - ) -> Rc { - Rc::new(Self { - time_dimensions, - dimensions, - time_dimensions_filters, - dimensions_filters, - measures_filters, - segments, - time_shifts: TimeShiftState::default(), - }) - } - - pub fn clone_state(&self) -> Self { - Self { - time_dimensions: self.time_dimensions.clone(), - dimensions: self.dimensions.clone(), - time_dimensions_filters: self.time_dimensions_filters.clone(), - dimensions_filters: self.dimensions_filters.clone(), - measures_filters: self.measures_filters.clone(), - segments: self.segments.clone(), - time_shifts: self.time_shifts.clone(), - } - } - - pub fn add_dimensions(&mut self, dimensions: Vec>) { - self.dimensions = self - .dimensions - .iter() - .cloned() - .chain(dimensions.into_iter()) - .unique_by(|d| d.clone().resolve_reference_chain().full_name()) - .collect_vec(); - } - - pub fn add_dimension_filter(&mut self, filter: FilterItem) { - self.dimensions_filters.push(filter); - } - - pub fn remove_multistage_dimensions( - &mut self, - resolved_dimensions: &HashSet, - ) -> Result<(), CubeError> { - let mut filtered = Vec::new(); - for d in &self.dimensions { - if resolved_dimensions.contains(&d.clone().resolve_reference_chain().full_name()) - || !has_multi_stage_members(&d, true)? - { - filtered.push(d.clone()); - } - } - self.dimensions = filtered; - let mut filtered = Vec::new(); - for d in &self.time_dimensions { - if resolved_dimensions.contains(&d.clone().resolve_reference_chain().full_name()) - || !has_multi_stage_members(&d, true)? - { - filtered.push(d.clone()); - } - } - self.time_dimensions = filtered; - Ok(()) - } - - pub fn add_time_shifts(&mut self, time_shifts: MeasureTimeShifts) -> Result<(), CubeError> { - let resolved_shifts = match time_shifts { - MeasureTimeShifts::Dimensions(dimensions) => dimensions, - MeasureTimeShifts::Common(interval) => self - .all_time_members() - .into_iter() - .map(|m| DimensionTimeShift { - interval: Some(interval.clone()), - dimension: m, - name: None, - }) - .collect_vec(), - MeasureTimeShifts::Named(named_shift) => self - .all_time_members() - .into_iter() - .map(|m| DimensionTimeShift { - interval: None, - dimension: m, - name: Some(named_shift.clone()), - }) - .collect_vec(), - }; - for ts in resolved_shifts.into_iter() { - if let Some(exists) = self - .time_shifts - .dimensions_shifts - .get_mut(&ts.dimension.full_name()) - { - if let Some(interval) = exists.interval.clone() { - if let Some(new_interval) = ts.interval { - exists.interval = Some(interval + new_interval); - } else { - return Err(CubeError::internal(format!( - "Cannot use both named ({}) and interval ({}) shifts for the same dimension: {}.", - ts.name.clone().unwrap_or("-".to_string()), - interval.to_sql(), - ts.dimension.full_name(), - ))); - } - } else if let Some(named_shift) = exists.name.clone() { - return if let Some(new_interval) = ts.interval { - Err(CubeError::internal(format!( - "Cannot use both named ({}) and interval ({}) shifts for the same dimension: {}.", - named_shift, - new_interval.to_sql(), - ts.dimension.full_name(), - ))) - } else { - Err(CubeError::internal(format!( - "Cannot use more than one named shifts ({}, {}) for the same dimension: {}.", - ts.name.clone().unwrap_or("-".to_string()), - named_shift, - ts.dimension.full_name(), - ))) - }; - } - } else { - self.time_shifts - .dimensions_shifts - .insert(ts.dimension.full_name(), ts); - } - } - Ok(()) - } - - pub fn time_shifts(&self) -> &TimeShiftState { - &self.time_shifts - } - - fn all_time_members(&self) -> Vec> { - let mut filter_symbols = self.all_dimensions_symbols(); - for filter_item in self - .time_dimensions_filters - .iter() - .chain(self.dimensions_filters.iter()) - .chain(self.segments.iter()) - { - filter_item.find_all_member_evaluators(&mut filter_symbols); - } - - let time_symbols = filter_symbols - .into_iter() - .filter_map(|m| { - let symbol = if let Ok(time_dim) = m.as_time_dimension() { - time_dim.base_symbol().clone().resolve_reference_chain() - } else { - m.resolve_reference_chain() - }; - if let Ok(dim) = symbol.as_dimension() { - if dim.is_time() { - Some(symbol) - } else { - None - } - } else { - None - } - }) - .unique_by(|s| s.full_name()) - .collect_vec(); - time_symbols - } - - pub fn time_dimensions_filters(&self) -> &Vec { - &self.time_dimensions_filters - } - - pub fn time_dimensions_symbols(&self) -> Vec> { - self.time_dimensions().clone() - } - - pub fn dimensions_symbols(&self) -> Vec> { - self.dimensions.clone() - } - - pub fn all_dimensions_symbols(&self) -> Vec> { - self.time_dimensions - .iter() - .cloned() - .chain(self.dimensions.iter().cloned()) - .collect() - } - - pub fn dimensions_filters(&self) -> &Vec { - &self.dimensions_filters - } - - pub fn segments(&self) -> &Vec { - &self.segments - } - - pub fn measures_filters(&self) -> &Vec { - &self.measures_filters - } - - pub fn dimensions(&self) -> &Vec> { - &self.dimensions - } - - pub fn time_dimensions(&self) -> &Vec> { - &self.time_dimensions - } - - pub fn set_time_dimensions(&mut self, time_dimensions: Vec>) { - self.time_dimensions = time_dimensions; - } - - pub fn set_dimensions(&mut self, dimensions: Vec>) { - self.dimensions = dimensions; - } - - pub fn remove_filter_for_member(&mut self, member_name: &String) { - self.time_dimensions_filters = - self.extract_filters_exclude_member(member_name, &self.time_dimensions_filters); - self.dimensions_filters = - self.extract_filters_exclude_member(member_name, &self.dimensions_filters); - self.measures_filters = - self.extract_filters_exclude_member(member_name, &self.measures_filters); - } - - fn extract_filters_exclude_member( - &self, - member_name: &String, - filters: &Vec, - ) -> Vec { - let mut result = Vec::new(); - for item in filters.iter() { - match item { - FilterItem::Group(group) => { - let new_group = FilterItem::Group(Rc::new(FilterGroup::new( - group.operator.clone(), - self.extract_filters_exclude_member(member_name, &group.items), - ))); - result.push(new_group); - } - FilterItem::Item(itm) => { - if &itm.member_name() != member_name { - result.push(FilterItem::Item(itm.clone())); - } - } - FilterItem::Segment(_) => {} - } - } - result - } - - pub fn has_filters_for_member(&self, member_name: &String) -> bool { - self.has_filters_for_member_impl(member_name, &self.time_dimensions_filters) - || self.has_filters_for_member_impl(member_name, &self.dimensions_filters) - || self.has_filters_for_member_impl(member_name, &self.measures_filters) - } - - fn has_filters_for_member_impl(&self, member_name: &String, filters: &Vec) -> bool { - for item in filters.iter() { - match item { - FilterItem::Group(group) => { - if self.has_filters_for_member_impl(member_name, &group.items) { - return true; - } - } - FilterItem::Item(itm) => { - if &itm.member_name() == member_name { - return true; - } - } - FilterItem::Segment(_) => {} - } - } - false - } - - /// Replace InDateRange filter with bounded version for rolling window without granularity. - /// Unlike `replace_regular_date_range_filter` which uses time_series CTE references, - /// this keeps parameter-based filters suitable for queries without a time_series CTE. - pub fn replace_date_range_for_rolling_window_without_granularity( - &mut self, - member_name: &String, - trailing: &Option, - leading: &Option, - ) -> Result<(), CubeError> { - let trailing_unbounded = trailing.as_deref() == Some("unbounded"); - let leading_unbounded = leading.as_deref() == Some("unbounded"); - - if !trailing_unbounded && !leading_unbounded { - return Ok(()); - } - - if trailing_unbounded && leading_unbounded { - // Both unbounded — remove the date range filter entirely - self.time_dimensions_filters.retain(|item| match item { - FilterItem::Item(itm) => { - !(&itm.member_name() == member_name - && matches!(itm.filter_operator(), FilterOperator::InDateRange)) - } - _ => true, - }); - } else if trailing_unbounded { - // Remove lower bound: InDateRange(from, to) → BeforeOrOnDate(to) - let mut new_filters = Vec::new(); - for item in self.time_dimensions_filters.iter() { - match item { - FilterItem::Item(itm) - if &itm.member_name() == member_name - && matches!(itm.filter_operator(), FilterOperator::InDateRange) => - { - let values = itm.values(); - let to_value = if values.len() >= 2 { - vec![values[1].clone()] - } else { - values.clone() - }; - new_filters.push(FilterItem::Item(itm.change_operator( - FilterOperator::BeforeOrOnDate, - to_value, - itm.use_raw_values(), - )?)); - } - other => new_filters.push(other.clone()), - } - } - self.time_dimensions_filters = new_filters; - } else { - // leading unbounded: remove upper bound: InDateRange(from, to) → AfterOrOnDate(from) - let mut new_filters = Vec::new(); - for item in self.time_dimensions_filters.iter() { - match item { - FilterItem::Item(itm) - if &itm.member_name() == member_name - && matches!(itm.filter_operator(), FilterOperator::InDateRange) => - { - let values = itm.values(); - let from_value = if !values.is_empty() { - vec![values[0].clone()] - } else { - values.clone() - }; - new_filters.push(FilterItem::Item(itm.change_operator( - FilterOperator::AfterOrOnDate, - from_value, - itm.use_raw_values(), - )?)); - } - other => new_filters.push(other.clone()), - } - } - self.time_dimensions_filters = new_filters; - } - Ok(()) - } - - pub fn replace_regular_date_range_filter( - &mut self, - member_name: &String, - left_interval: Option, - right_interval: Option, - ) -> Result<(), CubeError> { - let operator = FilterOperator::RegularRollingWindowDateRange; - let values = vec![left_interval.clone(), right_interval.clone()]; - self.time_dimensions_filters = self.change_date_range_filter_impl( - member_name, - &self.time_dimensions_filters, - &operator, - None, - &values, - &None, - )?; - Ok(()) - } - - pub fn replace_to_date_date_range_filter( - &mut self, - member_name: &String, - granularity: &String, - ) -> Result<(), CubeError> { - let operator = FilterOperator::ToDateRollingWindowDateRange; - let values = vec![Some(granularity.clone())]; - self.time_dimensions_filters = self.change_date_range_filter_impl( - member_name, - &self.time_dimensions_filters, - &operator, - None, - &values, - &None, - )?; - Ok(()) - } - - pub fn replace_range_in_date_filter( - &mut self, - member_name: &String, - new_from: String, - new_to: String, - ) -> Result<(), CubeError> { - let operator = FilterOperator::InDateRange; - let replacement_values = vec![Some(new_from), Some(new_to)]; - self.time_dimensions_filters = self.change_date_range_filter_impl( - member_name, - &self.time_dimensions_filters, - &operator, - None, - &vec![], - &Some(replacement_values), - )?; - Ok(()) - } - - pub fn replace_range_to_subquery_in_date_filter( - &mut self, - member_name: &String, - new_from: String, - new_to: String, - ) -> Result<(), CubeError> { - let operator = FilterOperator::InDateRange; - let replacement_values = vec![Some(new_from), Some(new_to)]; - self.time_dimensions_filters = self.change_date_range_filter_impl( - member_name, - &self.time_dimensions_filters, - &operator, - Some(true), - &vec![], - &Some(replacement_values), - )?; - Ok(()) - } - - fn change_date_range_filter_impl( - &self, - member_name: &String, - filters: &Vec, - operator: &FilterOperator, - use_raw_values: Option, - additional_values: &Vec>, - replacement_values: &Option>>, - ) -> Result, CubeError> { - let mut result = Vec::new(); - for item in filters.iter() { - match item { - FilterItem::Group(group) => { - let new_group = FilterItem::Group(Rc::new(FilterGroup::new( - group.operator.clone(), - self.change_date_range_filter_impl( - member_name, - filters, - operator, - use_raw_values, - additional_values, - replacement_values, - )?, - ))); - result.push(new_group); - } - FilterItem::Item(itm) => { - let itm = if &itm.member_name() == member_name - && matches!(itm.filter_operator(), FilterOperator::InDateRange) - { - let mut values = if let Some(values) = replacement_values { - values.clone() - } else { - itm.values().clone() - }; - values.extend(additional_values.iter().cloned()); - let use_raw_values = use_raw_values.unwrap_or(itm.use_raw_values()); - itm.change_operator(operator.clone(), values, use_raw_values)? - } else { - itm.clone() - }; - result.push(FilterItem::Item(itm)); - } - FilterItem::Segment(segment) => result.push(FilterItem::Segment(segment.clone())), - } - } - Ok(result) - } -} - -impl PartialEq for MultiStageAppliedState { - fn eq(&self, other: &Self) -> bool { - let dims_eq = self.dimensions.len() == other.dimensions.len() - && self - .dimensions - .iter() - .zip(other.dimensions.iter()) - .all(|(a, b)| { - a.clone().resolve_reference_chain().full_name() - == b.clone().resolve_reference_chain().full_name() - }) - && self - .time_dimensions - .iter() - .zip(other.time_dimensions.iter()) - .all(|(a, b)| { - a.clone().resolve_reference_chain().full_name() - == b.clone().resolve_reference_chain().full_name() - }); - dims_eq - && self.time_dimensions_filters == other.time_dimensions_filters - && self.dimensions_filters == other.dimensions_filters - && self.measures_filters == other.measures_filters - && self.time_shifts.dimensions_shifts == other.time_shifts.dimensions_shifts - } -} - -impl Debug for MultiStageAppliedState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MultiStageAppliedState") - .field( - "dimensions", - &self.dimensions.iter().map(|d| d.full_name()).join(", "), - ) - .field("time_shifts", &self.time_shifts) - .finish() - } -} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs index 1cb2d37ffb4d1..b19bad335bcc8 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/member_query_planner.rs @@ -3,7 +3,6 @@ use super::{ MultiStageQueryDescription, RollingWindowDescription, TimeSeriesDescription, }; use crate::logical_plan::*; -use crate::planner::join_hints::JoinHints; use crate::planner::planners::{multi_stage::RollingWindowType, QueryPlanner, SimpleQueryPlanner}; use crate::planner::query_tools::QueryTools; use crate::planner::GranularityHelper; @@ -59,26 +58,15 @@ impl MultiStageMemberQueryPlanner { &self, time_dimension: Rc, ) -> Result, CubeError> { - let cte_query_properties = QueryProperties::try_new_from_precompiled( - self.query_tools.clone(), - vec![], - vec![], - vec![time_dimension.clone()], - vec![], - vec![], - vec![], - vec![], - vec![], - None, - None, - true, - true, - false, - false, - Rc::new(JoinHints::new()), - true, - self.query_properties.disable_external_pre_aggregations(), - )?; + let cte_query_properties = QueryProperties::builder() + .query_tools(self.query_tools.clone()) + .time_dimensions(vec![time_dimension.clone()]) + .ignore_cumulative(true) + .ungrouped(true) + .disable_external_pre_aggregations( + self.query_properties.disable_external_pre_aggregations(), + ) + .build()?; let simple_query_planer = SimpleQueryPlanner::new(self.query_tools.clone(), cte_query_properties); @@ -213,8 +201,8 @@ impl MultiStageMemberQueryPlanner { vec![] }; let schema = LogicalSchema::default() - .set_dimensions(self.description.state().dimensions_symbols()) - .set_time_dimensions(self.description.state().time_dimensions_symbols()) + .set_dimensions(self.description.state().dimensions().clone()) + .set_time_dimensions(self.description.state().time_dimensions().clone()) .set_measures(measures) .into_rc(); @@ -271,8 +259,8 @@ impl MultiStageMemberQueryPlanner { &self, _multi_stage_member: &MultiStageInodeMember, ) -> Result, CubeError> { - let mut dimensions = self.description.state().dimensions_symbols(); - let mut time_dimensions = self.description.state().time_dimensions_symbols(); + let mut dimensions = self.description.state().dimensions().clone(); + let mut time_dimensions = self.description.state().time_dimensions().clone(); let mut measures = vec![]; let cte_member = self.description.member().evaluation_node(); match cte_member.as_ref() { @@ -353,8 +341,8 @@ impl MultiStageMemberQueryPlanner { fn plan_for_leaf_cte_query(&self) -> Result, CubeError> { let member_node = self.description.member_node(); - let mut dimensions = self.description.state().dimensions_symbols(); - let mut time_dimensions = self.description.state().time_dimensions_symbols(); + let mut dimensions = self.description.state().dimensions().clone(); + let mut time_dimensions = self.description.state().time_dimensions().clone(); let mut measures = vec![]; if !self.description.member().is_without_member_leaf() { match member_node.as_ref() { @@ -379,26 +367,23 @@ impl MultiStageMemberQueryPlanner { } } - let cte_query_properties = QueryProperties::try_new_from_precompiled( - self.query_tools.clone(), - measures, - dimensions, - time_dimensions, - self.description.state().time_dimensions_filters().clone(), - self.description.state().dimensions_filters().clone(), - self.description.state().measures_filters().clone(), - self.description.state().segments().clone(), - vec![], - None, - None, - true, - self.description.member().is_ungrupped(), - false, - false, - self.query_properties.query_join_hints().clone(), - false, - self.query_properties.disable_external_pre_aggregations(), - )?; + let cte_query_properties = QueryProperties::builder() + .query_tools(self.query_tools.clone()) + .measures(measures) + .dimensions(dimensions) + .time_dimensions(time_dimensions) + .time_dimensions_filters(self.description.state().time_dimensions_filters().clone()) + .dimensions_filters(self.description.state().dimensions_filters().clone()) + .measures_filters(self.description.state().measures_filters().clone()) + .segments(self.description.state().segments().clone()) + .ignore_cumulative(true) + .ungrouped(self.description.member().is_ungrupped()) + .query_join_hints(self.query_properties.query_join_hints().clone()) + .allow_multi_stage(false) + .disable_external_pre_aggregations( + self.query_properties.disable_external_pre_aggregations(), + ) + .build()?; let query_planner = QueryPlanner::new(cte_query_properties.clone(), self.query_tools.clone()); @@ -432,14 +417,14 @@ impl MultiStageMemberQueryPlanner { .description .input() .iter() - .flat_map(|descr| descr.state().dimensions_symbols().clone()) + .flat_map(|descr| descr.state().dimensions().iter().cloned()) .unique_by(|dim| dim.full_name()) .collect_vec(); let time_dimensions = self .description .input() .iter() - .flat_map(|descr| descr.state().time_dimensions_symbols().clone()) + .flat_map(|descr| descr.state().time_dimensions().iter().cloned()) .unique_by(|dim| dim.full_name()) .collect_vec(); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs index f38e10d10fa00..ef7a01c22fabc 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/mod.rs @@ -1,4 +1,3 @@ -mod applied_state; mod cte_state; mod member; mod member_query_planner; @@ -6,7 +5,6 @@ mod multi_stage_query_planner; mod query_description; mod time_shift_state; -pub use applied_state::*; pub use cte_state::CteState; pub use member::*; pub use member_query_planner::MultiStageMemberQueryPlanner; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs index ddd30e69680c3..9ebb0aab879a7 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/multi_stage_query_planner.rs @@ -1,6 +1,6 @@ use super::{ - CteState, MultiStageAppliedState, MultiStageInodeMember, MultiStageInodeMemberType, - MultiStageLeafMemberType, MultiStageMember, MultiStageMemberQueryPlanner, MultiStageMemberType, + CteState, MultiStageInodeMember, MultiStageInodeMemberType, MultiStageLeafMemberType, + MultiStageMember, MultiStageMemberQueryPlanner, MultiStageMemberType, MultiStageQueryDescription, RollingWindowDescription, TimeSeriesDescription, }; use crate::cube_bridge::measure_definition::RollingWindow; @@ -57,14 +57,20 @@ impl MultiStageQueryPlanner { } let mut descriptions = Vec::new(); - let state = MultiStageAppliedState::new( - self.query_properties.time_dimensions().clone(), - self.query_properties.dimensions().clone(), - self.query_properties.time_dimensions_filters().clone(), - self.query_properties.dimensions_filters().clone(), - vec![], //TODO: We do not pass measures filters to CTE queries. This seems correct, but we need to check - self.query_properties.segments().clone(), - ); + // Multi-stage CTE state: a query carrying the dimensions/filters of the + // current node in the multi-stage tree. measures_filters are + // intentionally dropped — CTE queries do not propagate them. order_by + // is set to an empty vec so the builder skips default_order: this + // value is used only as a state container, never planned directly. + let state = QueryProperties::builder() + .query_tools(self.query_tools.clone()) + .dimensions(self.query_properties.dimensions().clone()) + .time_dimensions(self.query_properties.time_dimensions().clone()) + .dimensions_filters(self.query_properties.dimensions_filters().clone()) + .time_dimensions_filters(self.query_properties.time_dimensions_filters().clone()) + .segments(self.query_properties.segments().clone()) + .order_by(Some(vec![])) + .build()?; let mut resolved_multi_stage_dimensions = HashSet::new(); @@ -156,7 +162,7 @@ impl MultiStageQueryPlanner { fn make_childs( &self, member: Rc, - new_state: Rc, + new_state: Rc, result: &mut Vec>, descriptions: &mut Vec>, resolved_multi_stage_dimensions: &mut HashSet, @@ -195,7 +201,7 @@ impl MultiStageQueryPlanner { fn default_make_childs( &self, member: Rc, - new_state: Rc, + new_state: Rc, result: &mut Vec>, descriptions: &mut Vec>, resolved_multi_stage_dimensions: &mut HashSet, @@ -242,7 +248,7 @@ impl MultiStageQueryPlanner { fn try_make_childs_for_case_switch( &self, case: &CaseSwitchDefinition, - new_state: Rc, + new_state: Rc, result: &mut Vec>, descriptions: &mut Vec>, resolved_multi_stage_dimensions: &mut HashSet, @@ -290,7 +296,7 @@ impl MultiStageQueryPlanner { } for (_, (dep, values)) in deps { - let mut state = new_state.clone_state(); + let mut state = new_state.as_ref().clone(); if let Some(values) = values { if !values.is_empty() { let filter = BaseFilter::try_new( @@ -319,7 +325,7 @@ impl MultiStageQueryPlanner { fn make_queries_descriptions( &self, member: Rc, - state: Rc, + state: Rc, descriptions: &mut Vec>, resolved_multi_stage_dimensions: &mut HashSet, cte_state: &mut CteState, @@ -327,7 +333,7 @@ impl MultiStageQueryPlanner { let member = member.resolve_reference_chain(); let member = apply_static_filter_to_symbol(&member, state.dimensions_filters())?; let state = if member.is_dimension() { - let mut new_state = state.clone_state(); + let mut new_state = state.as_ref().clone(); new_state.remove_multistage_dimensions(resolved_multi_stage_dimensions)?; Rc::new(new_state) } else { @@ -382,7 +388,7 @@ impl MultiStageQueryPlanner { || multi_stage_member.time_shift().is_some() || state.has_filters_for_member(&member_name) { - let mut new_state = state.clone_state(); + let mut new_state = state.as_ref().clone(); if !dimensions_to_add.is_empty() { new_state.add_dimensions(dimensions_to_add.clone()); } @@ -428,7 +434,7 @@ impl MultiStageQueryPlanner { pub fn try_plan_rolling_window( &self, member: Rc, - state: Rc, + state: Rc, descriptions: &mut Vec>, resolved_multi_stage_dimensions: &mut HashSet, cte_state: &mut CteState, @@ -601,7 +607,7 @@ impl MultiStageQueryPlanner { fn add_time_series_get_range_query( &self, time_dimension: Rc, - state: Rc, + state: Rc, descriptions: &mut Vec>, ) -> Result, CubeError> { let description = if let Some(description) = descriptions @@ -632,7 +638,7 @@ impl MultiStageQueryPlanner { fn add_time_series( &self, time_dimension: Rc, - state: Rc, + state: Rc, descriptions: &mut Vec>, ) -> Result, CubeError> { let description = if let Some(description) = @@ -678,7 +684,7 @@ impl MultiStageQueryPlanner { fn add_rolling_window_base( &self, member: Rc, - state: Rc, + state: Rc, ungrouped: bool, descriptions: &mut Vec>, cte_state: &mut CteState, @@ -727,9 +733,9 @@ impl MultiStageQueryPlanner { fn replace_date_range_for_rolling_window( &self, rolling_window: &RollingWindow, - state: Rc, - ) -> Result, CubeError> { - let mut new_state = state.clone_state(); + state: Rc, + ) -> Result, CubeError> { + let mut new_state = state.as_ref().clone(); for filter_item in state.time_dimensions_filters() { if let FilterItem::Item(filter) = filter_item { if matches!(filter.filter_operator(), FilterOperator::InDateRange) { @@ -748,11 +754,11 @@ impl MultiStageQueryPlanner { &self, time_dimension: Rc, rolling_window: &RollingWindow, - state: Rc, - ) -> Result<(Rc, Rc), CubeError> { + state: Rc, + ) -> Result<(Rc, Rc), CubeError> { let time_dimension_symbol = time_dimension.as_time_dimension()?; let time_dimension_base_name = time_dimension_symbol.base_symbol().full_name(); - let mut new_state = state.clone_state(); + let mut new_state = state.as_ref().clone(); let trailing_granularity = GranularityHelper::granularity_from_interval(&rolling_window.trailing); let leading_granularity = diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/query_description.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/query_description.rs index 311bf5b249269..c264ae10da684 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/query_description.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/query_description.rs @@ -1,36 +1,21 @@ -use super::{MultiStageAppliedState, MultiStageMember}; +use super::MultiStageMember; use crate::logical_plan::LogicalSchema; -use crate::planner::MemberSymbol; +use crate::planner::{MemberSymbol, QueryProperties}; use cubenativeutils::CubeError; use itertools::Itertools; -use std::fmt::Debug; use std::rc::Rc; pub struct MultiStageQueryDescription { member: Rc, - state: Rc, + state: Rc, input: Vec>, alias: String, } -impl Debug for MultiStageQueryDescription { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MultiStageQueryDescription") - .field( - "member_node", - &format!("node with path {}", self.member_node().full_name()), - ) - .field("state", &self.state) - .field("input", &self.input) - .field("alias", &self.alias) - .finish() - } -} - impl MultiStageQueryDescription { pub fn new( member: Rc, - state: Rc, + state: Rc, input: Vec>, alias: String, ) -> Rc { @@ -49,6 +34,7 @@ impl MultiStageQueryDescription { .set_measures(vec![self.member_node().clone()]) .into_rc() } + pub fn member_node(&self) -> &Rc { &self.member.evaluation_node() } @@ -61,8 +47,8 @@ impl MultiStageQueryDescription { &self.member } - pub fn state(&self) -> Rc { - self.state.clone() + pub fn state(&self) -> &Rc { + &self.state } pub fn member_name(&self) -> String { @@ -124,8 +110,8 @@ impl MultiStageQueryDescription { dimensions: &mut Vec>, time_dimensions: &mut Vec>, ) { - dimensions.extend(self.state.dimensions_symbols().iter().cloned()); - time_dimensions.extend(self.state.time_dimensions_symbols().iter().cloned()); + dimensions.extend(self.state.dimensions().iter().cloned()); + time_dimensions.extend(self.state.time_dimensions().iter().cloned()); for child in self.input.iter() { child.collect_all_non_multi_stage_dimension_impl(dimensions, time_dimensions); } @@ -134,8 +120,8 @@ impl MultiStageQueryDescription { pub fn is_match_member_and_state( &self, member_node: &Rc, - state: &Rc, + state: &Rc, ) -> bool { - member_node.full_name() == self.member_name() && state == &self.state + member_node.full_name() == self.member_name() && state.eq_as_state(&self.state) } } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/time_shift_state.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/time_shift_state.rs index b555053c8f9e0..921546ac8cf17 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/time_shift_state.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/time_shift_state.rs @@ -3,7 +3,7 @@ use crate::planner::DimensionTimeShift; use cubenativeutils::CubeError; use std::collections::HashMap; -#[derive(Clone, Default, Debug)] +#[derive(Clone, Default, Debug, PartialEq)] pub struct TimeShiftState { pub dimensions_shifts: HashMap, } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs index eac6cfc700249..97acff93c32e3 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/order_planner.rs @@ -19,10 +19,7 @@ impl OrderPlanner { ) } - pub fn custom_order( - order_by: &Vec, - members: &Vec>, - ) -> Vec { + pub fn custom_order(order_by: &[OrderByItem], members: &[Rc]) -> Vec { let mut result = Vec::new(); for itm in order_by.iter() { for found_item in members diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs index e844fab5bf0cf..bbd98b4cde749 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/planners/simple_query_planer.rs @@ -47,7 +47,7 @@ impl SimpleQueryPlanner { offset: self.query_properties.offset(), limit: self.query_properties.row_limit(), ungrouped: self.query_properties.ungrouped(), - order_by: self.query_properties.order_by().clone(), + order_by: self.query_properties.order_by().to_vec(), })) .source(source.into()) .build(); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs index 783133c3de3f1..32e141429022d 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties.rs @@ -1,26 +1,33 @@ -use super::filter::compiler::FilterCompiler; -use super::filter::BaseSegment; -use super::query_tools::QueryTools; -use crate::cube_bridge::member_expression::MemberExpressionExpressionDef; -use crate::planner::join_hints::JoinHints; -use crate::planner::GranularityHelper; -use crate::planner::{ - apply_static_filter_to_filter_item, apply_static_filter_to_symbol, MemberExpressionExpression, - MemberExpressionSymbol, TimeDimensionSymbol, -}; +//! `QueryProperties` describes what a query asks for: members, filters, +//! ordering and the planner flags that govern compilation. +//! +//! Construction goes through `QueryProperties::builder()`. Most fields default +//! to empty/false; callers spell out only what differs from a vanilla query. +//! For inputs that originate from `BaseQueryOptions`, see +//! [`QueryPropertiesCompiler`](super::query_properties_compiler). +use super::query_tools::QueryTools; use super::MemberSymbol; -use crate::cube_bridge::base_query_options::BaseQueryOptions; use crate::cube_bridge::join_definition::JoinDefinition; -use crate::cube_bridge::options_member::OptionsMember; use crate::planner::collectors::{collect_multiplied_measures, has_multi_stage_members}; -use crate::planner::filter::{Filter, FilterItem}; +use crate::planner::filter::{Filter, FilterGroup, FilterItem, FilterOperator}; +use crate::planner::join_hints::JoinHints; use crate::planner::multi_fact_join_groups::{MeasuresJoinHints, MultiFactJoinGroups}; +use crate::planner::planners::multi_stage::TimeShiftState; +use crate::planner::{ + apply_static_filter_to_filter_item, apply_static_filter_to_symbol, DimensionTimeShift, + MeasureTimeShifts, +}; use cubenativeutils::CubeError; use itertools::Itertools; +use std::cell::OnceCell; use std::collections::HashSet; use std::rc::Rc; +use typed_builder::TypedBuilder; +/// One entry of a query's ORDER BY clause. Equality follows the +/// reference-chain-resolved name of the member, matching the semantics used +/// for member equivalence elsewhere in `QueryProperties`. #[derive(Clone, Debug)] pub struct OrderByItem { member_evaluator: Rc, @@ -48,10 +55,25 @@ impl OrderByItem { } } +impl PartialEq for OrderByItem { + fn eq(&self, other: &Self) -> bool { + self.desc == other.desc && member_chain_eq(&self.member_evaluator, &other.member_evaluator) + } +} + +// Compare two member symbols by their reference-chain-resolved full name. +fn member_chain_eq(a: &Rc, b: &Rc) -> bool { + a.clone().resolve_reference_chain().full_name() + == b.clone().resolve_reference_chain().full_name() +} + +/// A measure paired with the cube it should be aggregated under. The bound +/// cube can differ from the measure's own cube for member expressions +/// referencing a dimension. #[derive(Debug, Clone)] pub struct MultipliedMeasure { measure: Rc, - cube_name: String, //May differ from cube_name of the measure for a member_expression that refers to a dimension. + cube_name: String, } impl MultipliedMeasure { @@ -68,6 +90,11 @@ impl MultipliedMeasure { } } +/// Measures classified by how the planner must compute them: directly +/// aggregated alongside the rest of the query, wrapped in a multiplied +/// subquery, or planned as a multi-stage CTE. `rendered_as_multiplied` +/// tracks symbols that were originally multiplied even after additivity +/// reclassification flips them into `regular_measures`. #[derive(Default, Clone, Debug)] pub struct FullKeyAggregateMeasures { pub multiplied_measures: Vec>, @@ -76,427 +103,89 @@ pub struct FullKeyAggregateMeasures { pub rendered_as_multiplied_measures: HashSet, } -impl FullKeyAggregateMeasures { - pub fn has_multiplied_measures(&self) -> bool { - !self.multiplied_measures.is_empty() - } - - pub fn has_multi_stage_measures(&self) -> bool { - !self.multi_stage_measures.is_empty() - } -} - -#[derive(Clone)] +/// The full description of a query: selected members, filters, ordering, +/// planner flags, plus a lazy cache of the join groups derived from those +/// members. Constructed via the typed builder; `build()` substitutes +/// `default_order` if `order_by` was not set, applies static filters, and +/// wraps the result in `Rc`. +/// +/// Two equality flavours: [`PartialEq`] compares all semantic fields; +/// [`eq_as_state`](Self::eq_as_state) compares only members, filters, +/// segments and time-shifts. +#[derive(Clone, TypedBuilder)] +#[builder(build_method(into = Result, CubeError>))] pub struct QueryProperties { + query_tools: Rc, + #[builder(default)] measures: Vec>, + #[builder(default)] dimensions: Vec>, + #[builder(default)] + time_dimensions: Vec>, + #[builder(setter(skip), default)] + time_shifts: TimeShiftState, + #[builder(default)] dimensions_filters: Vec, + #[builder(default)] time_dimensions_filters: Vec, + #[builder(default)] measures_filters: Vec, + #[builder(default)] segments: Vec, - time_dimensions: Vec>, - order_by: Vec, + /// `None` lets `From` substitute [`Self::default_order`]; `Some(vec)` is + /// used verbatim, including `Some(empty)`. + #[builder(default)] + order_by: Option>, + #[builder(default)] row_limit: Option, + #[builder(default)] offset: Option, - query_tools: Rc, + #[builder(default)] ignore_cumulative: bool, + #[builder(default)] ungrouped: bool, - multi_fact_join_groups: MultiFactJoinGroups, + #[builder(default)] pre_aggregation_query: bool, + #[builder(default)] total_query: bool, + #[builder(default = Rc::new(JoinHints::new()))] query_join_hints: Rc, + #[builder(default = true)] allow_multi_stage: bool, + #[builder(default)] disable_external_pre_aggregations: bool, + #[builder(default)] pre_aggregation_id: Option, + #[builder(setter(skip), default)] + multi_fact_join_groups: OnceCell, } -impl QueryProperties { - pub fn try_new( - query_tools: Rc, - options: Rc, - ) -> Result, CubeError> { - let evaluator_compiler_cell = query_tools.evaluator_compiler().clone(); - let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); - - let dimensions = if let Some(dimensions) = &options.dimensions()? { - dimensions - .iter() - .map(|d| match d { - OptionsMember::MemberName(member_name) => { - evaluator_compiler.add_dimension_evaluator(member_name.clone()) - } - OptionsMember::MemberExpression(member_expression) => { - let cube_name = - if let Some(name) = &member_expression.static_data().cube_name { - name.clone() - } else { - "".to_string() - }; - let name = - if let Some(name) = &member_expression.static_data().expression_name { - name.clone() - } else { - "".to_string() - }; - let expression_call = match member_expression.expression()? { - MemberExpressionExpressionDef::Sql(sql) => { - evaluator_compiler.compile_sql_call(&cube_name, sql)? - } - MemberExpressionExpressionDef::Struct(_) => { - return Err(CubeError::user(format!( - "Expression struct not supported for dimension" - ))); - } - }; - let cube_symbol = evaluator_compiler - .add_cube_table_evaluator(cube_name.clone(), vec![])?; - let member_expression_symbol = MemberExpressionSymbol::try_new( - cube_symbol, - name.clone(), - MemberExpressionExpression::SqlCall(expression_call), - member_expression.static_data().definition.clone(), - None, - vec![cube_name.clone()], - )?; - Ok(MemberSymbol::new_member_expression( - member_expression_symbol, - )) - } - }) - .collect::, _>>()? - } else { - Vec::new() - }; - - let time_dimensions = if let Some(time_dimensions) = &options.static_data().time_dimensions - { - time_dimensions - .iter() - .map(|d| -> Result, CubeError> { - let base_symbol = - evaluator_compiler.add_dimension_evaluator(d.dimension.clone())?; - let granularity_obj = GranularityHelper::make_granularity_obj( - query_tools.cube_evaluator().clone(), - &mut evaluator_compiler, - &base_symbol.cube_name(), - &base_symbol.name(), - d.granularity.clone(), - )?; - let date_range_tuple = if let Some(date_range) = &d.date_range { - assert_eq!(date_range.len(), 2); - Some((date_range[0].clone(), date_range[1].clone())) - } else { - None - }; - let symbol = MemberSymbol::new_time_dimension(TimeDimensionSymbol::new( - base_symbol, - d.granularity.clone(), - granularity_obj, - date_range_tuple, - )); - Ok(symbol) - }) - .collect::, _>>()? - } else { - Vec::new() - }; - - let measures = if let Some(measures) = &options.measures()? { - measures - .iter() - .map(|d| match d { - OptionsMember::MemberName(member_name) => { - evaluator_compiler.add_measure_evaluator(member_name.clone()) - } - OptionsMember::MemberExpression(member_expression) => { - let cube_name = - if let Some(name) = &member_expression.static_data().cube_name { - name.clone() - } else { - "".to_string() - }; - let name = - if let Some(name) = &member_expression.static_data().expression_name { - name.clone() - } else if let Some(name) = &member_expression.static_data().name { - format!("{}.{}", cube_name, name) - } else { - "".to_string() - }; - let expression = match member_expression.expression()? { - MemberExpressionExpressionDef::Sql(sql) => { - MemberExpressionExpression::SqlCall( - evaluator_compiler.compile_sql_call(&cube_name, sql)?, - ) - } - MemberExpressionExpressionDef::Struct(expr) => { - if expr.static_data().expression_type != "PatchMeasure" { - return Err(CubeError::user(format!("Only `PatchMeasure` type of memeber expression is supported"))); - } - - if let Some(source_measure) = &expr.static_data().source_measure { - - let new_measure_type = expr.static_data().replace_aggregation_type.clone(); - let mut filters_to_add = vec![]; - if let Some(add_filters) = expr.add_filters()? { - for filter in add_filters.iter() { - let node = evaluator_compiler.compile_sql_call(&cube_name, filter.sql()?)?; - filters_to_add.push(node); - } - } - let source_measure_compiled = evaluator_compiler.add_measure_evaluator(source_measure.clone())?; - let symbol = if let Ok(source_measure) = source_measure_compiled.as_measure() { - - let patched_measure = source_measure.new_patched(new_measure_type, filters_to_add)?; - MemberSymbol::new_measure(patched_measure) - } else { - source_measure_compiled - }; - MemberExpressionExpression::PatchedSymbol(symbol) - - } else { - return Err(CubeError::user(format!("Source measure is required for `PatchMeasure` type of memeber expression"))); - } - - } - }; - let cube_symbol = evaluator_compiler.add_cube_table_evaluator(cube_name.clone(), vec![])?; - let member_expression_symbol = MemberExpressionSymbol::try_new( - cube_symbol, - name.clone(), - expression, - member_expression.static_data().definition.clone(), - None, - vec![cube_name.clone()], - )?; - Ok(MemberSymbol::new_member_expression(member_expression_symbol)) - } - }) - .collect::, _>>()? - /* measures - .iter() - .map(|m| { - let evaluator = evaluator_compiler.add_measure_evaluator(m.clone())?; - BaseMeasure::try_new_required(evaluator, query_tools.clone()) - }) - .collect::, _>>()? */ - } else { - Vec::new() - }; - - let segments = if let Some(segments) = &options.segments()? { - segments - .iter() - .map(|d| -> Result<_, CubeError> { - let segment = match d { - OptionsMember::MemberName(member_name) => { - let mut iter = query_tools - .cube_evaluator() - .parse_path("segments".to_string(), member_name.clone())? - .into_iter(); - let cube_name = iter.next().unwrap(); - let name = iter.next().unwrap(); - let definition = query_tools - .cube_evaluator() - .segment_by_path(member_name.clone())?; - let expression_evaluator = evaluator_compiler - .compile_sql_call(&cube_name, definition.sql()?)?; - let cube_symbol = evaluator_compiler - .add_cube_table_evaluator(cube_name.clone(), vec![])?; - BaseSegment::try_new( - expression_evaluator, - cube_symbol, - name, - Some(member_name.clone()), - ) - } - OptionsMember::MemberExpression(member_expression) => { - let cube_name = - if let Some(name) = &member_expression.static_data().cube_name { - name.clone() - } else { - "".to_string() - }; - let name = if let Some(name) = - &member_expression.static_data().expression_name - { - name.clone() - } else { - "".to_string() - }; - let expression_evaluator = match member_expression.expression()? { - MemberExpressionExpressionDef::Sql(sql) => { - evaluator_compiler.compile_sql_call(&cube_name, sql)? - } - MemberExpressionExpressionDef::Struct(_) => { - return Err(CubeError::user(format!( - "Expression struct not supported for dimension" - ))); - } - }; - let cube_symbol = evaluator_compiler - .add_cube_table_evaluator(cube_name.clone(), vec![])?; - BaseSegment::try_new(expression_evaluator, cube_symbol, name, None) - } - }?; - Ok(FilterItem::Segment(segment)) - }) - .collect::, _>>()? - } else { - Vec::new() - }; - - let mut filter_compiler = FilterCompiler::new(&mut evaluator_compiler, query_tools.clone()); - if let Some(filters) = &options.static_data().filters { - for filter in filters { - filter_compiler.add_item(filter)?; - } +/// Finalize the builder output. Wired into +/// `QueryProperties::builder().…build()` via `build_method(into = …)` — +/// not intended for direct `.into()` conversions, which would re-apply +/// the finalization steps. +impl From for Result, CubeError> { + fn from(mut qp: QueryProperties) -> Self { + if qp.order_by.is_none() { + qp.order_by = Some(QueryProperties::default_order( + &qp.dimensions, + &qp.time_dimensions, + &qp.measures, + )); } - for time_dimension in &time_dimensions { - filter_compiler.add_time_dimension_item(time_dimension)?; - } - let (dimensions_filters, time_dimensions_filters, measures_filters) = - filter_compiler.extract_result(); - - //FIXME may be this filter should be applied on other place - let time_dimensions = time_dimensions - .into_iter() - .filter(|dim| { - if let Ok(td) = dim.as_time_dimension() { - td.has_granularity() - } else { - true - } - }) - .collect_vec(); - - let order_by = if let Some(order) = &options.static_data().order { - order - .iter() - .map(|o| -> Result<_, CubeError> { - let evaluator = if let Some(found) = - dimensions.iter().find(|d| d.name() == o.id) - { - found.clone() - } else if let Some(found) = time_dimensions.iter().find(|d| d.name() == o.id) { - found.clone() - } else if let Some(found) = measures.iter().find(|d| d.name() == o.id) { - found.clone() - } else { - evaluator_compiler.add_auto_resolved_member_evaluator(o.id.clone())? - }; - Ok(OrderByItem::new(evaluator, o.is_desc())) - }) - .collect::, _>>()? - } else { - Self::default_order(&dimensions, &time_dimensions, &measures) - }; - - let row_limit = if let Some(row_limit) = &options.static_data().row_limit { - row_limit.parse::().ok() - } else { - None - }; - let offset = if let Some(offset) = &options.static_data().offset { - offset.parse::().ok() - } else { - None - }; - let ungrouped = options.static_data().ungrouped.unwrap_or(false); - - let query_join_hints = Rc::new(JoinHints::from_items( - options.join_hints()?.unwrap_or_default(), - )); - - let pre_aggregation_query = options.static_data().pre_aggregation_query.unwrap_or(false); - let total_query = options.static_data().total_query.unwrap_or(false); - let disable_external_pre_aggregations = - options.static_data().disable_external_pre_aggregations; - let pre_aggregation_id = options.static_data().pre_aggregation_id.clone(); - - let mut res = Self { - measures, - dimensions, - segments, - time_dimensions, - time_dimensions_filters, - dimensions_filters, - measures_filters, - order_by, - row_limit, - offset, - multi_fact_join_groups: MultiFactJoinGroups::empty(query_tools.clone()), - query_tools, - ignore_cumulative: false, - ungrouped, - pre_aggregation_query, - total_query, - query_join_hints, - allow_multi_stage: true, - disable_external_pre_aggregations, - pre_aggregation_id, - }; - res.apply_static_filters()?; - Ok(Rc::new(res)) - } - - pub fn try_new_from_precompiled( - query_tools: Rc, - measures: Vec>, - dimensions: Vec>, - time_dimensions: Vec>, - time_dimensions_filters: Vec, - dimensions_filters: Vec, - measures_filters: Vec, - segments: Vec, - order_by: Vec, - row_limit: Option, - offset: Option, - ignore_cumulative: bool, - ungrouped: bool, - pre_aggregation_query: bool, - total_query: bool, - query_join_hints: Rc, - allow_multi_stage: bool, - disable_external_pre_aggregations: bool, - ) -> Result, CubeError> { - let order_by = if order_by.is_empty() { - Self::default_order(&dimensions, &time_dimensions, &measures) - } else { - order_by - }; - - let mut res = Self { - measures, - dimensions, - time_dimensions, - time_dimensions_filters, - dimensions_filters, - segments, - measures_filters, - order_by, - row_limit, - offset, - multi_fact_join_groups: MultiFactJoinGroups::empty(query_tools.clone()), - query_tools, - ignore_cumulative, - ungrouped, - pre_aggregation_query, - total_query, - query_join_hints, - allow_multi_stage, - disable_external_pre_aggregations, - pre_aggregation_id: None, - }; - res.apply_static_filters()?; - - Ok(Rc::new(res)) + qp.apply_static_filters()?; + Ok(Rc::new(qp)) } +} +impl QueryProperties { pub fn allow_multi_stage(&self) -> bool { self.allow_multi_stage } + // Push every entry of `dimensions_filters` into matching `case` + // expressions on each member, filter and order item. Run once at + // construction; mutators do not re-apply it. fn apply_static_filters(&mut self) -> Result<(), CubeError> { let dimensions_filters = self.dimensions_filters.clone(); for dim in self.dimensions.iter_mut() { @@ -520,11 +209,14 @@ impl QueryProperties { for filter_item in self.segments.iter_mut() { *filter_item = apply_static_filter_to_filter_item(filter_item, &dimensions_filters)?; } - for order_item in self.order_by.iter_mut() { + for order_item in self.order_by.iter_mut().flatten() { order_item.member_evaluator = apply_static_filter_to_symbol(&order_item.member_evaluator, &dimensions_filters)?; } + Ok(()) + } + fn compute_multi_fact_join_groups(&self) -> Result { let measures_join_hints = MeasuresJoinHints::builder(&self.query_join_hints) .add_dimensions(&self.dimensions) .add_dimensions(&self.extract_dimensions_from_order()) @@ -533,16 +225,22 @@ impl QueryProperties { .add_filters(&self.dimensions_filters) .add_filters(&self.segments) .build(&self.all_used_measures()?)?; - self.multi_fact_join_groups = - MultiFactJoinGroups::try_new(self.query_tools.clone(), measures_join_hints)?; - Ok(()) + MultiFactJoinGroups::try_new(self.query_tools.clone(), measures_join_hints) + } + + fn multi_fact_join_groups(&self) -> Result<&MultiFactJoinGroups, CubeError> { + if let Some(g) = self.multi_fact_join_groups.get() { + return Ok(g); + } + let computed = self.compute_multi_fact_join_groups()?; + Ok(self.multi_fact_join_groups.get_or_init(move || computed)) } pub fn compute_join_multi_fact_groups_with_measures( &self, measures: &[Rc], ) -> Result { - self.multi_fact_join_groups.for_measures(measures) + self.multi_fact_join_groups()?.for_measures(measures) } pub fn is_total_query(&self) -> bool { @@ -552,6 +250,7 @@ impl QueryProperties { fn extract_dimensions_from_order(&self) -> Vec> { self.order_by .iter() + .flatten() .filter_map(|order| { if order.member_evaluator.as_dimension().is_ok() { Some(order.member_evaluator.clone()) @@ -562,12 +261,12 @@ impl QueryProperties { .collect() } - pub fn is_multi_fact_join(&self) -> bool { - self.multi_fact_join_groups.is_multi_fact() + fn is_multi_fact_join(&self) -> Result { + Ok(self.multi_fact_join_groups()?.is_multi_fact()) } pub fn simple_query_join(&self) -> Result>, CubeError> { - self.multi_fact_join_groups.single_join() + self.multi_fact_join_groups()?.single_join() } pub fn measures(&self) -> &Vec> { @@ -582,6 +281,10 @@ impl QueryProperties { &self.time_dimensions } + pub fn time_shifts(&self) -> &TimeShiftState { + &self.time_shifts + } + pub fn time_dimensions_filters(&self) -> &Vec { &self.time_dimensions_filters } @@ -606,13 +309,8 @@ impl QueryProperties { self.offset } - pub fn order_by(&self) -> &Vec { - &self.order_by - } - - pub fn set_order_by_to_default(&mut self) { - self.order_by = - Self::default_order(&self.dimensions, &self.time_dimensions, &self.measures); + pub fn order_by(&self) -> &[OrderByItem] { + self.order_by.as_deref().unwrap_or(&[]) } pub fn ungrouped(&self) -> bool { @@ -631,6 +329,8 @@ impl QueryProperties { self.pre_aggregation_id.as_deref() } + /// Concatenation of `time_dimensions_filters`, `dimensions_filters`, and + /// `segments` into a single `Filter`. `measures_filters` are not included. pub fn all_filters(&self) -> Option { let items = self .time_dimensions_filters @@ -664,6 +364,9 @@ impl QueryProperties { } } + /// Every symbol the query touches: selected members, members referenced + /// inside filters, and measures pulled in by measure-filter or order-by + /// references. Deduplicated by full name. pub fn all_used_symbols(&self) -> Result>, CubeError> { let mut members = vec![]; members.extend(self.time_dimensions.iter().cloned()); @@ -705,7 +408,7 @@ impl QueryProperties { .collect_vec() } - pub fn fill_all_filter_symbols(&self, members: &mut Vec>) { + fn fill_all_filter_symbols(&self, members: &mut Vec>) { if let Some(all_filters) = self.all_filters() { for filter_item in all_filters.items.iter() { filter_item.find_all_member_evaluators(members); @@ -713,10 +416,13 @@ impl QueryProperties { } } + /// First time-dimension with a granularity (ASC) if any; otherwise the + /// first measure (DESC) when both measures and dimensions are present; + /// otherwise the first dimension (ASC). Empty when none apply. pub fn default_order( - dimensions: &Vec>, - time_dimensions: &Vec>, - measures: &Vec>, + dimensions: &[Rc], + time_dimensions: &[Rc], + measures: &[Rc], ) -> Vec { let mut result = Vec::new(); if let Some(granularity_dim) = time_dimensions.iter().find(|d| { @@ -739,7 +445,7 @@ impl QueryProperties { let full_aggregate_measure = self.full_key_aggregate_measures()?; if full_aggregate_measure.multiplied_measures.is_empty() && (full_aggregate_measure.multi_stage_measures.is_empty() || !self.allow_multi_stage) - && !self.is_multi_fact_join() + && !self.is_multi_fact_join()? && (!self.has_multi_stage_dimensions()? || !self.allow_multi_stage) { Ok(true) @@ -826,7 +532,7 @@ impl QueryProperties { for item in self.measures_filters.iter() { self.fill_missed_measures_from_filter(item, &mut measures)?; } - for item in self.order_by.iter() { + for item in self.order_by.iter().flatten() { if let Ok(measure) = item.member_evaluator.as_measure() { if !measures .iter() @@ -860,4 +566,519 @@ impl QueryProperties { } Ok(()) } + + // Compare two member slices element-wise by reference-chain-resolved + // full name. + fn members_equivalent(a: &[Rc], b: &[Rc]) -> bool { + a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| member_chain_eq(x, y)) + } + + // --- members / filters / time-shifts mutators --- + // + // Each mutator below changes a field that feeds into the join-groups + // computation and invalidates the lazy cache. + + pub fn set_dimensions(&mut self, dimensions: Vec>) { + self.dimensions = dimensions; + self.invalidate_join_groups_cache(); + } + + pub fn set_time_dimensions(&mut self, time_dimensions: Vec>) { + self.time_dimensions = time_dimensions; + self.invalidate_join_groups_cache(); + } + + /// Append `dimensions` to the existing list, deduplicating by + /// reference-chain-resolved full name. + pub fn add_dimensions(&mut self, dimensions: Vec>) { + self.dimensions = self + .dimensions + .iter() + .cloned() + .chain(dimensions.into_iter()) + .unique_by(|d| d.clone().resolve_reference_chain().full_name()) + .collect_vec(); + self.invalidate_join_groups_cache(); + } + + pub fn add_dimension_filter(&mut self, filter: FilterItem) { + self.dimensions_filters.push(filter); + self.invalidate_join_groups_cache(); + } + + /// Keep only dimensions and time-dimensions whose chain-resolved name is + /// in `resolved_dimensions` or that have no multi-stage members. + pub fn remove_multistage_dimensions( + &mut self, + resolved_dimensions: &HashSet, + ) -> Result<(), CubeError> { + let mut filtered = Vec::new(); + for d in &self.dimensions { + if resolved_dimensions.contains(&d.clone().resolve_reference_chain().full_name()) + || !has_multi_stage_members(d, true)? + { + filtered.push(d.clone()); + } + } + self.dimensions = filtered; + let mut filtered = Vec::new(); + for d in &self.time_dimensions { + if resolved_dimensions.contains(&d.clone().resolve_reference_chain().full_name()) + || !has_multi_stage_members(d, true)? + { + filtered.push(d.clone()); + } + } + self.time_dimensions = filtered; + self.invalidate_join_groups_cache(); + Ok(()) + } + + /// Merge `time_shifts` into [`Self::time_shifts`]. Interval shifts on + /// the same dimension compose; mixing a named shift with an interval + /// (or two named shifts) on the same dimension returns an error. + pub fn add_time_shifts(&mut self, time_shifts: MeasureTimeShifts) -> Result<(), CubeError> { + let resolved_shifts = match time_shifts { + MeasureTimeShifts::Dimensions(dimensions) => dimensions, + MeasureTimeShifts::Common(interval) => self + .all_time_members() + .into_iter() + .map(|m| DimensionTimeShift { + interval: Some(interval.clone()), + dimension: m, + name: None, + }) + .collect_vec(), + MeasureTimeShifts::Named(named_shift) => self + .all_time_members() + .into_iter() + .map(|m| DimensionTimeShift { + interval: None, + dimension: m, + name: Some(named_shift.clone()), + }) + .collect_vec(), + }; + for ts in resolved_shifts.into_iter() { + if let Some(exists) = self + .time_shifts + .dimensions_shifts + .get_mut(&ts.dimension.full_name()) + { + if let Some(interval) = exists.interval.clone() { + if let Some(new_interval) = ts.interval { + exists.interval = Some(interval + new_interval); + } else { + return Err(CubeError::internal(format!( + "Cannot use both named ({}) and interval ({}) shifts for the same dimension: {}.", + ts.name.clone().unwrap_or("-".to_string()), + interval.to_sql(), + ts.dimension.full_name(), + ))); + } + } else if let Some(named_shift) = exists.name.clone() { + return if let Some(new_interval) = ts.interval { + Err(CubeError::internal(format!( + "Cannot use both named ({}) and interval ({}) shifts for the same dimension: {}.", + named_shift, + new_interval.to_sql(), + ts.dimension.full_name(), + ))) + } else { + Err(CubeError::internal(format!( + "Cannot use more than one named shifts ({}, {}) for the same dimension: {}.", + ts.name.clone().unwrap_or("-".to_string()), + named_shift, + ts.dimension.full_name(), + ))) + }; + } + } else { + self.time_shifts + .dimensions_shifts + .insert(ts.dimension.full_name(), ts); + } + } + Ok(()) + } + + fn all_time_members(&self) -> Vec> { + let mut filter_symbols: Vec> = self + .dimensions + .iter() + .cloned() + .chain(self.time_dimensions.iter().cloned()) + .collect(); + for filter_item in self + .time_dimensions_filters + .iter() + .chain(self.dimensions_filters.iter()) + .chain(self.segments.iter()) + { + filter_item.find_all_member_evaluators(&mut filter_symbols); + } + filter_symbols + .into_iter() + .filter_map(|m| { + let symbol = if let Ok(time_dim) = m.as_time_dimension() { + time_dim.base_symbol().clone().resolve_reference_chain() + } else { + m.resolve_reference_chain() + }; + if let Ok(dim) = symbol.as_dimension() { + if dim.is_time() { + Some(symbol) + } else { + None + } + } else { + None + } + }) + .unique_by(|s| s.full_name()) + .collect_vec() + } + + pub fn remove_filter_for_member(&mut self, member_name: &str) { + self.time_dimensions_filters = + Self::extract_filters_exclude_member(member_name, &self.time_dimensions_filters); + self.dimensions_filters = + Self::extract_filters_exclude_member(member_name, &self.dimensions_filters); + self.measures_filters = + Self::extract_filters_exclude_member(member_name, &self.measures_filters); + self.invalidate_join_groups_cache(); + } + + fn extract_filters_exclude_member( + member_name: &str, + filters: &[FilterItem], + ) -> Vec { + let mut result = Vec::new(); + for item in filters.iter() { + match item { + FilterItem::Group(group) => { + let new_group = FilterItem::Group(Rc::new(FilterGroup::new( + group.operator.clone(), + Self::extract_filters_exclude_member(member_name, &group.items), + ))); + result.push(new_group); + } + FilterItem::Item(itm) => { + if itm.member_name() != member_name { + result.push(FilterItem::Item(itm.clone())); + } + } + FilterItem::Segment(_) => {} + } + } + result + } + + pub fn has_filters_for_member(&self, member_name: &str) -> bool { + Self::has_filters_for_member_impl(member_name, &self.time_dimensions_filters) + || Self::has_filters_for_member_impl(member_name, &self.dimensions_filters) + || Self::has_filters_for_member_impl(member_name, &self.measures_filters) + } + + fn has_filters_for_member_impl(member_name: &str, filters: &[FilterItem]) -> bool { + for item in filters.iter() { + match item { + FilterItem::Group(group) => { + if Self::has_filters_for_member_impl(member_name, &group.items) { + return true; + } + } + FilterItem::Item(itm) => { + if itm.member_name() == member_name { + return true; + } + } + FilterItem::Segment(_) => {} + } + } + false + } + + /// Rewrite an `InDateRange` filter on `member_name` according to the + /// trailing/leading bounds: both `unbounded` removes the filter entirely; + /// trailing-`unbounded` rewrites to `BeforeOrOnDate(to)`; leading- + /// `unbounded` rewrites to `AfterOrOnDate(from)`. Other inputs are + /// no-ops. + pub fn replace_date_range_for_rolling_window_without_granularity( + &mut self, + member_name: &str, + trailing: &Option, + leading: &Option, + ) -> Result<(), CubeError> { + let trailing_unbounded = trailing.as_deref() == Some("unbounded"); + let leading_unbounded = leading.as_deref() == Some("unbounded"); + + if !trailing_unbounded && !leading_unbounded { + return Ok(()); + } + + if trailing_unbounded && leading_unbounded { + // Both unbounded — remove the date range filter entirely + self.time_dimensions_filters.retain(|item| match item { + FilterItem::Item(itm) => { + !(itm.member_name() == member_name + && matches!(itm.filter_operator(), FilterOperator::InDateRange)) + } + _ => true, + }); + } else if trailing_unbounded { + // Remove lower bound: InDateRange(from, to) → BeforeOrOnDate(to) + let mut new_filters = Vec::new(); + for item in self.time_dimensions_filters.iter() { + match item { + FilterItem::Item(itm) + if itm.member_name() == member_name + && matches!(itm.filter_operator(), FilterOperator::InDateRange) => + { + let values = itm.values(); + let to_value = if values.len() >= 2 { + vec![values[1].clone()] + } else { + values.clone() + }; + new_filters.push(FilterItem::Item(itm.change_operator( + FilterOperator::BeforeOrOnDate, + to_value, + itm.use_raw_values(), + )?)); + } + other => new_filters.push(other.clone()), + } + } + self.time_dimensions_filters = new_filters; + } else { + // leading unbounded: remove upper bound: InDateRange(from, to) → AfterOrOnDate(from) + let mut new_filters = Vec::new(); + for item in self.time_dimensions_filters.iter() { + match item { + FilterItem::Item(itm) + if itm.member_name() == member_name + && matches!(itm.filter_operator(), FilterOperator::InDateRange) => + { + let values = itm.values(); + let from_value = if !values.is_empty() { + vec![values[0].clone()] + } else { + values.clone() + }; + new_filters.push(FilterItem::Item(itm.change_operator( + FilterOperator::AfterOrOnDate, + from_value, + itm.use_raw_values(), + )?)); + } + other => new_filters.push(other.clone()), + } + } + self.time_dimensions_filters = new_filters; + } + self.invalidate_join_groups_cache(); + Ok(()) + } + + pub fn replace_regular_date_range_filter( + &mut self, + member_name: &str, + left_interval: Option, + right_interval: Option, + ) -> Result<(), CubeError> { + let operator = FilterOperator::RegularRollingWindowDateRange; + let values = vec![left_interval.clone(), right_interval.clone()]; + self.time_dimensions_filters = self.change_date_range_filter_impl( + member_name, + &self.time_dimensions_filters, + &operator, + None, + &values, + &None, + )?; + self.invalidate_join_groups_cache(); + Ok(()) + } + + pub fn replace_to_date_date_range_filter( + &mut self, + member_name: &str, + granularity: &String, + ) -> Result<(), CubeError> { + let operator = FilterOperator::ToDateRollingWindowDateRange; + let values = vec![Some(granularity.clone())]; + self.time_dimensions_filters = self.change_date_range_filter_impl( + member_name, + &self.time_dimensions_filters, + &operator, + None, + &values, + &None, + )?; + self.invalidate_join_groups_cache(); + Ok(()) + } + + pub fn replace_range_in_date_filter( + &mut self, + member_name: &str, + new_from: String, + new_to: String, + ) -> Result<(), CubeError> { + let operator = FilterOperator::InDateRange; + let replacement_values = vec![Some(new_from), Some(new_to)]; + self.time_dimensions_filters = self.change_date_range_filter_impl( + member_name, + &self.time_dimensions_filters, + &operator, + None, + &vec![], + &Some(replacement_values), + )?; + self.invalidate_join_groups_cache(); + Ok(()) + } + + /// Same as [`replace_range_in_date_filter`](Self::replace_range_in_date_filter) + /// but forces the rewritten filter to use raw (unparametrized) values. + pub fn replace_range_to_subquery_in_date_filter( + &mut self, + member_name: &str, + new_from: String, + new_to: String, + ) -> Result<(), CubeError> { + let operator = FilterOperator::InDateRange; + let replacement_values = vec![Some(new_from), Some(new_to)]; + self.time_dimensions_filters = self.change_date_range_filter_impl( + member_name, + &self.time_dimensions_filters, + &operator, + Some(true), + &vec![], + &Some(replacement_values), + )?; + self.invalidate_join_groups_cache(); + Ok(()) + } + + fn change_date_range_filter_impl( + &self, + member_name: &str, + filters: &[FilterItem], + operator: &FilterOperator, + use_raw_values: Option, + additional_values: &Vec>, + replacement_values: &Option>>, + ) -> Result, CubeError> { + let mut result = Vec::new(); + for item in filters.iter() { + match item { + FilterItem::Group(group) => { + let new_group = FilterItem::Group(Rc::new(FilterGroup::new( + group.operator.clone(), + self.change_date_range_filter_impl( + member_name, + &group.items, + operator, + use_raw_values, + additional_values, + replacement_values, + )?, + ))); + result.push(new_group); + } + FilterItem::Item(itm) => { + let itm = if itm.member_name() == member_name + && matches!(itm.filter_operator(), FilterOperator::InDateRange) + { + let mut values = if let Some(values) = replacement_values { + values.clone() + } else { + itm.values().clone() + }; + values.extend(additional_values.iter().cloned()); + let use_raw_values = use_raw_values.unwrap_or(itm.use_raw_values()); + itm.change_operator(operator.clone(), values, use_raw_values)? + } else { + itm.clone() + }; + result.push(FilterItem::Item(itm)); + } + FilterItem::Segment(segment) => result.push(FilterItem::Segment(segment.clone())), + } + } + Ok(result) + } + + fn invalidate_join_groups_cache(&mut self) { + self.multi_fact_join_groups = OnceCell::new(); + } + + /// Equality over members (chain-resolved), the three filter slots, + /// segments and time-shifts. Excludes ordering, limits, planner flags + /// and join hints; for those fields use the full [`PartialEq`]. + pub fn eq_as_state(&self, other: &Self) -> bool { + Self::members_equivalent(&self.dimensions, &other.dimensions) + && Self::members_equivalent(&self.time_dimensions, &other.time_dimensions) + && self.dimensions_filters == other.dimensions_filters + && self.time_dimensions_filters == other.time_dimensions_filters + && self.measures_filters == other.measures_filters + && self.segments == other.segments + && self.time_shifts == other.time_shifts + } +} + +/// Equality over every semantic field. Members are compared by reference- +/// chain-resolved name; `query_tools` and the `multi_fact_join_groups` cache +/// are excluded. See also [`eq_as_state`](QueryProperties::eq_as_state). +impl PartialEq for QueryProperties { + fn eq(&self, other: &Self) -> bool { + // Destructure to fail compilation if a new field is added without an + // explicit decision about whether it participates in semantic equality. + let Self { + measures, + dimensions, + time_dimensions, + dimensions_filters, + time_dimensions_filters, + measures_filters, + segments, + time_shifts, + order_by, + row_limit, + offset, + ungrouped, + ignore_cumulative, + pre_aggregation_query, + total_query, + allow_multi_stage, + disable_external_pre_aggregations, + pre_aggregation_id, + query_join_hints, + // Not part of semantic equality: + query_tools: _, + multi_fact_join_groups: _, + } = self; + + Self::members_equivalent(measures, &other.measures) + && Self::members_equivalent(dimensions, &other.dimensions) + && Self::members_equivalent(time_dimensions, &other.time_dimensions) + && *dimensions_filters == other.dimensions_filters + && *time_dimensions_filters == other.time_dimensions_filters + && *measures_filters == other.measures_filters + && *segments == other.segments + && *time_shifts == other.time_shifts + && *order_by == other.order_by + && *row_limit == other.row_limit + && *offset == other.offset + && *ungrouped == other.ungrouped + && *ignore_cumulative == other.ignore_cumulative + && *pre_aggregation_query == other.pre_aggregation_query + && *total_query == other.total_query + && *allow_multi_stage == other.allow_multi_stage + && *disable_external_pre_aggregations == other.disable_external_pre_aggregations + && *pre_aggregation_id == other.pre_aggregation_id + && *query_join_hints == other.query_join_hints + } } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties_compiler.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties_compiler.rs new file mode 100644 index 0000000000000..f6c29bdcb24c1 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/query_properties_compiler.rs @@ -0,0 +1,441 @@ +//! Translates [`BaseQueryOptions`] into a finalized [`QueryProperties`] — +//! resolves member/segment/filter/order references against the cube +//! evaluator and folds them into the typed builder. + +use std::rc::Rc; + +use cubenativeutils::CubeError; +use itertools::Itertools; + +use crate::cube_bridge::base_query_options::BaseQueryOptions; +use crate::cube_bridge::member_expression::{ + MemberExpressionDefinition, MemberExpressionExpressionDef, +}; +use crate::cube_bridge::options_member::OptionsMember; + +use super::filter::compiler::FilterCompiler; +use super::filter::{BaseSegment, FilterItem}; +use super::join_hints::JoinHints; +use super::query_properties::{OrderByItem, QueryProperties}; +use super::query_tools::QueryTools; +use super::{ + Compiler, GranularityHelper, MemberExpressionExpression, MemberExpressionSymbol, MemberSymbol, + TimeDimensionSymbol, +}; + +/// One-shot translator from [`BaseQueryOptions`] into a finalized +/// [`QueryProperties`]. +pub struct QueryPropertiesCompiler { + query_tools: Rc, +} + +impl QueryPropertiesCompiler { + pub fn new(query_tools: Rc) -> Self { + Self { query_tools } + } + + pub fn build( + self, + options: Rc, + ) -> Result, CubeError> { + let options = options.as_ref(); + let evaluator_compiler_cell = self.query_tools.evaluator_compiler().clone(); + let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); + + let dimensions = self.compile_dimensions(&mut evaluator_compiler, options)?; + let time_dimensions_raw = self.compile_time_dimensions(&mut evaluator_compiler, options)?; + let measures = self.compile_measures(&mut evaluator_compiler, options)?; + let segments = self.compile_segments(&mut evaluator_compiler, options)?; + + let (dimensions_filters, time_dimensions_filters, measures_filters) = + self.compile_filters(&mut evaluator_compiler, options, &time_dimensions_raw)?; + + // FIXME may be this filter should be applied on other place + let time_dimensions = Self::filter_time_dimensions_with_granularity(time_dimensions_raw); + + let order_by = self.compile_order_by( + &mut evaluator_compiler, + options, + &dimensions, + &time_dimensions, + &measures, + )?; + + let row_limit = options + .static_data() + .row_limit + .as_ref() + .and_then(|v| v.parse::().ok()); + let offset = options + .static_data() + .offset + .as_ref() + .and_then(|v| v.parse::().ok()); + let ungrouped = options.static_data().ungrouped.unwrap_or(false); + let pre_aggregation_query = options.static_data().pre_aggregation_query.unwrap_or(false); + let total_query = options.static_data().total_query.unwrap_or(false); + let disable_external_pre_aggregations = + options.static_data().disable_external_pre_aggregations; + let pre_aggregation_id = options.static_data().pre_aggregation_id.clone(); + + let query_join_hints = Rc::new(JoinHints::from_items( + options.join_hints()?.unwrap_or_default(), + )); + + QueryProperties::builder() + .query_tools(self.query_tools) + .measures(measures) + .dimensions(dimensions) + .time_dimensions(time_dimensions) + .time_dimensions_filters(time_dimensions_filters) + .dimensions_filters(dimensions_filters) + .measures_filters(measures_filters) + .segments(segments) + .order_by(order_by) + .row_limit(row_limit) + .offset(offset) + .ungrouped(ungrouped) + .pre_aggregation_query(pre_aggregation_query) + .total_query(total_query) + .query_join_hints(query_join_hints) + .disable_external_pre_aggregations(disable_external_pre_aggregations) + .pre_aggregation_id(pre_aggregation_id) + .build() + } + + fn compile_dimensions( + &self, + evaluator_compiler: &mut Compiler, + options: &dyn BaseQueryOptions, + ) -> Result>, CubeError> { + let Some(dimensions) = options.dimensions()? else { + return Ok(Vec::new()); + }; + dimensions + .iter() + .map(|d| match d { + OptionsMember::MemberName(member_name) => { + evaluator_compiler.add_dimension_evaluator(member_name.clone()) + } + OptionsMember::MemberExpression(member_expression) => { + Self::compile_member_expression_dimension(evaluator_compiler, member_expression) + } + }) + .collect() + } + + // Struct expressions are rejected; only SQL calls are accepted here. + fn compile_member_expression_dimension( + evaluator_compiler: &mut Compiler, + member_expression: &Rc, + ) -> Result, CubeError> { + let cube_name = member_expression + .static_data() + .cube_name + .clone() + .unwrap_or_default(); + let name = member_expression + .static_data() + .expression_name + .clone() + .unwrap_or_default(); + let expression_call = match member_expression.expression()? { + MemberExpressionExpressionDef::Sql(sql) => { + evaluator_compiler.compile_sql_call(&cube_name, sql)? + } + MemberExpressionExpressionDef::Struct(_) => { + return Err(CubeError::user( + "Expression struct not supported for dimension".to_string(), + )); + } + }; + let cube_symbol = evaluator_compiler.add_cube_table_evaluator(cube_name.clone(), vec![])?; + let member_expression_symbol = MemberExpressionSymbol::try_new( + cube_symbol, + name, + MemberExpressionExpression::SqlCall(expression_call), + member_expression.static_data().definition.clone(), + None, + vec![cube_name], + )?; + Ok(MemberSymbol::new_member_expression( + member_expression_symbol, + )) + } + + fn compile_time_dimensions( + &self, + evaluator_compiler: &mut Compiler, + options: &dyn BaseQueryOptions, + ) -> Result>, CubeError> { + let Some(time_dimensions) = &options.static_data().time_dimensions else { + return Ok(Vec::new()); + }; + time_dimensions + .iter() + .map(|d| -> Result, CubeError> { + let base_symbol = + evaluator_compiler.add_dimension_evaluator(d.dimension.clone())?; + let granularity_obj = GranularityHelper::make_granularity_obj( + self.query_tools.cube_evaluator().clone(), + evaluator_compiler, + &base_symbol.cube_name(), + &base_symbol.name(), + d.granularity.clone(), + )?; + let date_range_tuple = if let Some(date_range) = &d.date_range { + assert_eq!(date_range.len(), 2); + Some((date_range[0].clone(), date_range[1].clone())) + } else { + None + }; + Ok(MemberSymbol::new_time_dimension(TimeDimensionSymbol::new( + base_symbol, + d.granularity.clone(), + granularity_obj, + date_range_tuple, + ))) + }) + .collect() + } + + fn compile_measures( + &self, + evaluator_compiler: &mut Compiler, + options: &dyn BaseQueryOptions, + ) -> Result>, CubeError> { + let Some(measures) = options.measures()? else { + return Ok(Vec::new()); + }; + measures + .iter() + .map(|d| match d { + OptionsMember::MemberName(member_name) => { + evaluator_compiler.add_measure_evaluator(member_name.clone()) + } + OptionsMember::MemberExpression(member_expression) => { + Self::compile_member_expression_measure(evaluator_compiler, member_expression) + } + }) + .collect() + } + + // Accepts either a SQL call or a `PatchMeasure` struct; other struct + // expression types are rejected. + fn compile_member_expression_measure( + evaluator_compiler: &mut Compiler, + member_expression: &Rc, + ) -> Result, CubeError> { + let static_data = member_expression.static_data(); + let cube_name = static_data.cube_name.clone().unwrap_or_default(); + let name = if let Some(name) = &static_data.expression_name { + name.clone() + } else if let Some(name) = &static_data.name { + format!("{}.{}", cube_name, name) + } else { + String::new() + }; + let expression = match member_expression.expression()? { + MemberExpressionExpressionDef::Sql(sql) => MemberExpressionExpression::SqlCall( + evaluator_compiler.compile_sql_call(&cube_name, sql)?, + ), + MemberExpressionExpressionDef::Struct(expr) => { + if expr.static_data().expression_type != "PatchMeasure" { + return Err(CubeError::user( + "Only `PatchMeasure` type of member expression is supported".to_string(), + )); + } + + let Some(source_measure) = &expr.static_data().source_measure else { + return Err(CubeError::user( + "Source measure is required for `PatchMeasure` type of member expression" + .to_string(), + )); + }; + + let new_measure_type = expr.static_data().replace_aggregation_type.clone(); + let mut filters_to_add = vec![]; + if let Some(add_filters) = expr.add_filters()? { + for filter in add_filters.iter() { + let node = + evaluator_compiler.compile_sql_call(&cube_name, filter.sql()?)?; + filters_to_add.push(node); + } + } + let source_measure_compiled = + evaluator_compiler.add_measure_evaluator(source_measure.clone())?; + let symbol = if let Ok(source_measure) = source_measure_compiled.as_measure() { + let patched_measure = + source_measure.new_patched(new_measure_type, filters_to_add)?; + MemberSymbol::new_measure(patched_measure) + } else { + source_measure_compiled + }; + MemberExpressionExpression::PatchedSymbol(symbol) + } + }; + let cube_symbol = evaluator_compiler.add_cube_table_evaluator(cube_name.clone(), vec![])?; + let member_expression_symbol = MemberExpressionSymbol::try_new( + cube_symbol, + name, + expression, + static_data.definition.clone(), + None, + vec![cube_name], + )?; + Ok(MemberSymbol::new_member_expression( + member_expression_symbol, + )) + } + + fn compile_segments( + &self, + evaluator_compiler: &mut Compiler, + options: &dyn BaseQueryOptions, + ) -> Result, CubeError> { + let Some(segments) = options.segments()? else { + return Ok(Vec::new()); + }; + segments + .iter() + .map(|d| -> Result<_, CubeError> { + let segment = match d { + OptionsMember::MemberName(member_name) => { + self.compile_named_segment(evaluator_compiler, member_name)? + } + OptionsMember::MemberExpression(member_expression) => { + Self::compile_member_expression_segment( + evaluator_compiler, + member_expression, + )? + } + }; + Ok(FilterItem::Segment(segment)) + }) + .collect() + } + + fn compile_named_segment( + &self, + evaluator_compiler: &mut Compiler, + member_name: &str, + ) -> Result, CubeError> { + let mut iter = self + .query_tools + .cube_evaluator() + .parse_path("segments".to_string(), member_name.to_string())? + .into_iter(); + let cube_name = iter.next().unwrap(); + let name = iter.next().unwrap(); + let definition = self + .query_tools + .cube_evaluator() + .segment_by_path(member_name.to_string())?; + let expression_evaluator = + evaluator_compiler.compile_sql_call(&cube_name, definition.sql()?)?; + let cube_symbol = evaluator_compiler.add_cube_table_evaluator(cube_name, vec![])?; + BaseSegment::try_new( + expression_evaluator, + cube_symbol, + name, + Some(member_name.to_string()), + ) + } + + fn compile_member_expression_segment( + evaluator_compiler: &mut Compiler, + member_expression: &Rc, + ) -> Result, CubeError> { + let cube_name = member_expression + .static_data() + .cube_name + .clone() + .unwrap_or_default(); + let name = member_expression + .static_data() + .expression_name + .clone() + .unwrap_or_default(); + let expression_evaluator = match member_expression.expression()? { + MemberExpressionExpressionDef::Sql(sql) => { + evaluator_compiler.compile_sql_call(&cube_name, sql)? + } + MemberExpressionExpressionDef::Struct(_) => { + return Err(CubeError::user( + "Expression struct not supported for segment".to_string(), + )); + } + }; + let cube_symbol = evaluator_compiler.add_cube_table_evaluator(cube_name, vec![])?; + BaseSegment::try_new(expression_evaluator, cube_symbol, name, None) + } + + // Returns `(dimension_filters, time_dimension_filters, measure_filters)`. + // Includes both the explicit `options.filters` entries and the implicit + // `dateRange` filter carried by each time dimension. + fn compile_filters( + &self, + evaluator_compiler: &mut Compiler, + options: &dyn BaseQueryOptions, + time_dimensions: &[Rc], + ) -> Result<(Vec, Vec, Vec), CubeError> { + let mut filter_compiler = FilterCompiler::new(evaluator_compiler, self.query_tools.clone()); + if let Some(filters) = &options.static_data().filters { + for filter in filters { + filter_compiler.add_item(filter)?; + } + } + for time_dimension in time_dimensions { + filter_compiler.add_time_dimension_item(time_dimension)?; + } + Ok(filter_compiler.extract_result()) + } + + // Drop time-dimension symbols that have no granularity. Non-time- + // dimension symbols pass through unchanged. + fn filter_time_dimensions_with_granularity( + time_dimensions: Vec>, + ) -> Vec> { + time_dimensions + .into_iter() + .filter(|dim| { + if let Ok(td) = dim.as_time_dimension() { + td.has_granularity() + } else { + true + } + }) + .collect_vec() + } + + // Returns `None` when `options.order` is absent, `Some(translated)` + // otherwise — including `Some(empty)` if the input was an empty array. + fn compile_order_by( + &self, + evaluator_compiler: &mut Compiler, + options: &dyn BaseQueryOptions, + dimensions: &[Rc], + time_dimensions: &[Rc], + measures: &[Rc], + ) -> Result>, CubeError> { + let Some(order) = &options.static_data().order else { + return Ok(None); + }; + let translated = order + .iter() + .map(|o| -> Result<_, CubeError> { + let evaluator = if let Some(found) = dimensions.iter().find(|d| d.name() == o.id) { + found.clone() + } else if let Some(found) = time_dimensions.iter().find(|d| d.name() == o.id) { + found.clone() + } else if let Some(found) = measures.iter().find(|d| d.name() == o.id) { + found.clone() + } else { + evaluator_compiler.add_auto_resolved_member_evaluator(o.id.clone())? + }; + Ok(OrderByItem::new(evaluator, o.is_desc())) + }) + .collect::, _>>()?; + Ok(Some(translated)) + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs index 1916c211d5505..8636f430eb810 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_templates/plan.rs @@ -3,6 +3,7 @@ use crate::cube_bridge::driver_tools::DriverTools; use crate::cube_bridge::sql_templates_render::SqlTemplatesRender; use crate::physical_plan::join::JoinType; use crate::planner::sql_templates::structs::TemplateCalcGroup; +use crate::utils::sql_expression_scanner::is_top_level_compound; use convert_case::{Boundary, Case, Casing}; use cubenativeutils::CubeError; use minijinja::context; @@ -53,7 +54,12 @@ impl PlanSqlTemplates { } pub fn convert_tz(&self, field: String) -> Result { - self.driver_tools.convert_tz(field) + let safe = if is_top_level_compound(&field) { + format!("({field})") + } else { + field + }; + self.driver_tools.convert_tz(safe) } pub fn is_external(&self) -> bool { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_custom_granularity.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_custom_granularity.yaml index 9c75b1880cb7d..0843e75236ee6 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_custom_granularity.yaml +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_custom_granularity.yaml @@ -25,6 +25,14 @@ cubes: - name: fiscal_year interval: "1 year" offset: "1 month" + + - name: fiscal_year_alias + type: time + sql: "{created_at.fiscal_year}" + + - name: created_at_minus_one_day + type: time + sql: "{CUBE}.created_at - interval '1 day'" measures: - name: count type: count diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs index ec8f0a1fbb69d..37db81dfd5c38 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/test_utils/test_context.rs @@ -11,7 +11,7 @@ use crate::planner::filter::Filter; use crate::planner::query_tools::QueryTools; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::top_level_planner::TopLevelPlanner; -use crate::planner::{GranularityHelper, QueryProperties}; +use crate::planner::{GranularityHelper, QueryProperties, QueryPropertiesCompiler}; use crate::planner::{MemberSymbol, TimeDimensionSymbol}; use crate::test_fixtures::cube_bridge::yaml::YamlBaseQueryOptions; use crate::test_fixtures::cube_bridge::{ @@ -368,7 +368,7 @@ impl TestContext { pub fn create_query_properties(&self, yaml: &str) -> Result, CubeError> { let options = self.create_query_options_from_yaml(yaml); - QueryProperties::try_new(self.query_tools.clone(), options) + QueryPropertiesCompiler::new(self.query_tools.clone()).build(options) } #[allow(dead_code)] @@ -382,7 +382,7 @@ impl TestContext { &self, options: Rc, ) -> Result { - let request = QueryProperties::try_new(self.query_tools.clone(), options)?; + let request = QueryPropertiesCompiler::new(self.query_tools.clone()).build(options)?; let planner = TopLevelPlanner::new(request, self.query_tools.clone(), true); let (sql, _) = planner.plan()?; Ok(sql) @@ -394,7 +394,7 @@ impl TestContext { ) -> Result<(String, Vec), cubenativeutils::CubeError> { let options = self.create_query_options_from_yaml(query); let ctx = self.for_options(options.as_ref())?; - let request = QueryProperties::try_new(ctx.query_tools.clone(), options)?; + let request = QueryPropertiesCompiler::new(ctx.query_tools.clone()).build(options)?; let planner = TopLevelPlanner::new(request, ctx.query_tools.clone(), true); planner.plan() } @@ -421,7 +421,8 @@ impl TestContext { let ctx = self .for_options(options.as_ref()) .expect("Failed to create context"); - let request = QueryProperties::try_new(ctx.query_tools.clone(), options) + let request = QueryPropertiesCompiler::new(ctx.query_tools.clone()) + .build(options) .expect("Failed to create query properties"); let planner = TopLevelPlanner::new(request, ctx.query_tools.clone(), true); let (raw_sql, pre_aggregations) = planner.plan().expect("Failed to plan query"); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/common_sql_generation.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/common_sql_generation.rs index af6a4d53333af..faa1aa45c5a92 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/common_sql_generation.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/common_sql_generation.rs @@ -1,6 +1,7 @@ -use crate::test_fixtures::cube_bridge::MockSchema; +use crate::test_fixtures::cube_bridge::{members_from_strings, MockBaseQueryOptions, MockSchema}; use crate::test_fixtures::test_utils::TestContext; use indoc::indoc; +use std::rc::Rc; #[test] fn test_member_to_alias() { @@ -410,3 +411,30 @@ async fn test_segment_with_subquery_dimension_in_view_with_dimension() { insta::assert_snapshot!(result); } } + +#[test] +fn test_explicit_empty_order_omits_order_by_clause() { + let schema = MockSchema::from_yaml_file("common/diamond_joins.yaml"); + let test_context = TestContext::new(schema).unwrap(); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(test_context.query_tools().cube_evaluator().clone()) + .base_tools(test_context.query_tools().base_tools().clone()) + .join_graph(test_context.query_tools().join_graph().clone()) + .security_context(test_context.security_context().clone()) + .measures(Some(members_from_strings(vec!["cube_a.count"]))) + .dimensions(Some(members_from_strings(vec!["cube_c.code"]))) + .order(Some(vec![])) + .build(), + ); + + let sql = test_context + .build_sql_from_options(options) + .expect("Should generate SQL with explicit empty order"); + + assert!( + !sql.to_uppercase().contains("ORDER BY"), + "ORDER BY should be absent when order is explicitly empty, got: {sql}" + ); +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/custom_granularities.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/custom_granularities.rs index 9b4357da337f1..8694f856f8a9b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/custom_granularities.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/custom_granularities.rs @@ -133,6 +133,40 @@ async fn test_half_year_with_sum_measure() { } } +#[tokio::test(flavor = "multi_thread")] +async fn test_type_time_alias_wraps_compound_exprs_before_tz_cast() { + let ctx = create_context(); + + let query = indoc! {" + measures: + - orders.count + dimensions: + - orders.fiscal_year_alias + - orders.created_at_minus_one_day + order: + - id: orders.fiscal_year_alias + convert_tz_for_raw_time_dimension: true + "}; + + let sql = ctx.build_sql(query).unwrap(); + + // Two independent precedence-trap fingerprints — `::` latching onto a + // trailing interval literal instead of the wrapped composed expression. + for bad in [ + "interval '1 month'::timestamptz", + "interval '1 day'::timestamptz", + ] { + assert!( + !sql.contains(bad), + "cast-precedence trap detected: {bad}\nFull SQL:\n{sql}" + ); + } + + if let Some(result) = ctx.try_execute_pg(query, SEED).await { + insta::assert_snapshot!(result); + } +} + #[tokio::test(flavor = "multi_thread")] async fn test_custom_granularity_with_daterange_filter() { let ctx = create_context(); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__custom_granularities__type_time_alias_wraps_compound_exprs_before_tz_cast.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__custom_granularities__type_time_alias_wraps_compound_exprs_before_tz_cast.snap new file mode 100644 index 0000000000000..45c8d90002e74 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__custom_granularities__type_time_alias_wraps_compound_exprs_before_tz_cast.snap @@ -0,0 +1,22 @@ +--- +source: cubesqlplanner/cubesqlplanner/src/tests/integration/custom_granularities.rs +assertion_line: 166 +expression: result +--- +orders__fiscal_year_alias | orders__created_at_minus_one_day | orders__count +--------------------------+----------------------------------+-------------- +2023-02-01 00:00:00 | 2024-01-14 10:00:00 | 1 +2024-02-01 00:00:00 | 2024-04-30 14:00:00 | 1 +2024-02-01 00:00:00 | 2024-06-14 08:00:00 | 1 +2024-02-01 00:00:00 | 2024-03-19 09:00:00 | 1 +2024-02-01 00:00:00 | 2025-01-19 10:00:00 | 1 +2024-02-01 00:00:00 | 2024-09-19 11:00:00 | 1 +2024-02-01 00:00:00 | 2024-11-14 14:00:00 | 1 +2024-02-01 00:00:00 | 2024-10-04 09:00:00 | 1 +2024-02-01 00:00:00 | 2024-08-09 10:00:00 | 1 +2024-02-01 00:00:00 | 2024-11-30 08:00:00 | 1 +2024-02-01 00:00:00 | 2024-02-09 11:00:00 | 1 +2025-02-01 00:00:00 | 2025-11-19 08:00:00 | 1 +2025-02-01 00:00:00 | 2025-07-14 09:00:00 | 1 +2025-02-01 00:00:00 | 2025-04-09 11:00:00 | 1 +2025-02-01 00:00:00 | 2025-08-31 14:00:00 | 1