diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 12bf993d5b9f9..070342511a6f1 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -19,6 +19,7 @@ import { ResultArrayWrapper, ResultMultiWrapper, ResultWrapper, + rowsToColumnar, } from '@cubejs-backend/native'; import type { Application as ExpressApplication, @@ -129,40 +130,6 @@ function systemAsyncHandler(handler: (req: Request & { context: ExtendedRequestC }; } -function rowsToColumnar(rawData: any): { members: string[]; columns: any[][] } { - let rows: any[]; - - if (Array.isArray(rawData)) { - rows = rawData; - } else if (rawData) { - rows = Array.from(rawData as Iterable); - } else { - rows = []; - } - - const rowCount = rows.length; - if (rowCount === 0) { - return { members: [], columns: [] }; - } - - const members = Object.keys(rows[0]); - const memberCount = members.length; - const columns: any[][] = new Array(memberCount); - - for (let j = 0; j < memberCount; j++) { - const member = members[j]; - const col = new Array(rowCount); - - for (let i = 0; i < rowCount; i++) { - col[i] = rows[i][member]; - } - - columns[j] = col; - } - - return { members, columns }; -} - // Prepared CheckAuthFn, default or from config: always async type PreparedCheckAuthFn = (ctx: any, authorization?: string) => Promise<{ securityContext: any; diff --git a/packages/cubejs-backend-native/js/ResultWrapper.ts b/packages/cubejs-backend-native/js/ResultWrapper.ts index 1fdc4a97eefcf..c5c65992d0bb4 100644 --- a/packages/cubejs-backend-native/js/ResultWrapper.ts +++ b/packages/cubejs-backend-native/js/ResultWrapper.ts @@ -15,6 +15,45 @@ export interface DataResult { getResults(): ResultWrapper[]; } +export interface JsRawColumnarData { + members: string[]; + columns: any[][]; +} + +export function rowsToColumnar(rawData: any): JsRawColumnarData { + let rows: any[]; + + if (Array.isArray(rawData)) { + rows = rawData; + } else if (rawData) { + rows = Array.from(rawData as Iterable); + } else { + rows = []; + } + + const rowCount = rows.length; + if (rowCount === 0) { + return { members: [], columns: [] }; + } + + const members = Object.keys(rows[0]); + const memberCount = members.length; + const columns: any[][] = new Array(memberCount); + + for (let j = 0; j < memberCount; j++) { + const member = members[j]; + const col = new Array(rowCount); + + for (let i = 0; i < rowCount; i++) { + col[i] = rows[i][member]; + } + + columns[j] = col; + } + + return { members, columns }; +} + class BaseWrapper { public readonly isWrapper: boolean = true; } @@ -140,14 +179,19 @@ export class ResultWrapper extends BaseWrapper implements DataResult { return [this.nativeReference]; } + // Pivot to columnar before serializing: the row-oriented form repeats + // every column name on every row, which inflates JSON size and forces + // the Rust side to allocate a per-row map before transposing back to + // its native columnar `QueryResult` representation. + // // Serialize to a Buffer so the Rust side can decode via - // serde_json::from_slice instead of walking a JsArray through the + // serde_json::from_slice instead of walking a JsValue through the // Neon bridge with JsValueDeserializer. On 5 MB of AoO rows - // (~21k rows × 8 fields) the JsArray walk costs ~80 ms locally; + // (~21k rows × 8 fields) the JsValue walk costs ~80 ms locally; // Buffer + serde_json is ~7× faster (M3 MAX) and tracks V8's JSON.parse // (~11 ms on the same payload). On a real server it should be 3-6× slower, - // so avoiding the JsArray walk matters even more there. - return [Buffer.from(JSON.stringify(this.jsResult))]; + // so avoiding the JsValue walk matters even more there. + return [Buffer.from(JSON.stringify(rowsToColumnar(this.jsResult)))]; } public setTransformData(td: any) { diff --git a/packages/cubejs-backend-native/src/orchestrator.rs b/packages/cubejs-backend-native/src/orchestrator.rs index 96905c387f7b8..447d3909578bd 100644 --- a/packages/cubejs-backend-native/src/orchestrator.rs +++ b/packages/cubejs-backend-native/src/orchestrator.rs @@ -5,7 +5,7 @@ use cubeorchestrator::query_result_transform::{ DBResponsePrimitive, DBResponseValue, RequestResultData, RequestResultDataMulti, TransformedData, }; -use cubeorchestrator::transport::{JsRawData, TransformDataRequest}; +use cubeorchestrator::transport::{JsRawColumnarData, TransformDataRequest}; use cubesql::compile::engine::df::scan::{ColumnarValueObject, FieldValue, ValueObject}; use cubesql::CubeError; use neon::context::{Context, FunctionContext, ModuleContext}; @@ -87,41 +87,27 @@ impl ResultWrapper { let raw_data_js = raw_data_js_arr.first().unwrap(); - let query_result = - if let Ok(js_box) = raw_data_js.downcast::>, _>(cx) { - Arc::clone(&js_box) - } else if let Ok(js_buffer) = raw_data_js.downcast::(cx) { - let bytes = js_buffer.as_slice(cx); - let js_raw_data: JsRawData = serde_json::from_slice(bytes).map_err(|e| { - CubeError::internal(format!( - "Can't parse raw data JSON from JS ResultWrapper: {}", - e - )) - })?; - - QueryResult::from_js_raw_data(js_raw_data) - .map(Arc::new) - .map_cube_err("Can't build results data from JS rawData")? - } else if let Ok(js_array) = raw_data_js.downcast::(cx) { - let deserializer = JsValueDeserializer::new(cx, js_array.upcast()); - let js_raw_data: JsRawData = match Deserialize::deserialize(deserializer) { - Ok(data) => data, - Err(_) => { - return Err(CubeError::internal( - "Can't deserialize results raw data from JS ResultWrapper object" - .to_string(), - )); - } - }; + let query_result = if let Ok(js_box) = + raw_data_js.downcast::>, _>(cx) + { + Arc::clone(&js_box) + } else if let Ok(js_buffer) = raw_data_js.downcast::(cx) { + let bytes = js_buffer.as_slice(cx); + let js_raw_data: JsRawColumnarData = serde_json::from_slice(bytes).map_err(|e| { + CubeError::internal(format!( + "Can't parse raw data JSON from JS ResultWrapper: {}", + e + )) + })?; - QueryResult::from_js_raw_data(js_raw_data) - .map(Arc::new) - .map_cube_err("Can't build results data from JS rawData")? - } else { - return Err(CubeError::internal( - "Can't deserialize results raw data from JS ResultWrapper object".to_string(), - )); - }; + QueryResult::from_js_raw_data(js_raw_data) + .map(Arc::new) + .map_cube_err("Can't build results data from JS rawData")? + } else { + return Err(CubeError::internal( + "Can't deserialize results raw data from JS ResultWrapper object".to_string(), + )); + }; Ok(Self { transform_data: transform_request, @@ -315,21 +301,14 @@ fn extract_query_result( Ok(Arc::clone(&js_box)) } else if let Ok(js_buffer) = data_arg.downcast::(cx) { let bytes = js_buffer.as_slice(cx); - let js_raw_data: JsRawData = serde_json::from_slice(bytes)?; - - QueryResult::from_js_raw_data(js_raw_data) - .map(Arc::new) - .map_err(anyhow::Error::from) - } else if let Ok(js_array) = data_arg.downcast::(cx) { - let deserializer = JsValueDeserializer::new(cx, js_array.upcast()); - let js_raw_data: JsRawData = Deserialize::deserialize(deserializer)?; + let js_raw_data: JsRawColumnarData = serde_json::from_slice(bytes)?; QueryResult::from_js_raw_data(js_raw_data) .map(Arc::new) .map_err(anyhow::Error::from) } else { Err(anyhow::anyhow!( - "Second argument must be an Array of JsBox> or JsArray" + "Second argument must be a JsBox> or a JsBuffer with columnar JsRawColumnarData JSON" )) } } @@ -357,7 +336,7 @@ pub fn get_cubestore_result(mut cx: FunctionContext) -> JsResult { for (i, row) in result.rows.iter().enumerate() { let js_row = cx.execute_scoped(|mut cx| { let js_row = JsObject::new(&mut cx); - for (key, value) in result.columns.iter().zip(row.iter()) { + for (key, value) in result.members.iter().zip(row.iter()) { let js_key = cx.string(key); let js_value: Handle<'_, JsValue> = match value { DBResponseValue::Primitive(DBResponsePrimitive::Null) => cx.null().upcast(), diff --git a/rust/cube/cubeorchestrator/benches/transform.rs b/rust/cube/cubeorchestrator/benches/transform.rs index a841cbc945a41..ab9959aa1c287 100644 --- a/rust/cube/cubeorchestrator/benches/transform.rs +++ b/rust/cube/cubeorchestrator/benches/transform.rs @@ -5,10 +5,9 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Through use cubeorchestrator::query_message_parser::QueryResult; use cubeorchestrator::query_result_transform::{DBResponsePrimitive, TransformedData}; use cubeorchestrator::transport::{ - ConfigItem, JsRawData, MemberOrMemberExpression, NormalizedQuery, QueryType, ResultType, - TransformDataRequest, + ConfigItem, JsRawColumnarData, MemberOrMemberExpression, NormalizedQuery, QueryType, + ResultType, TransformDataRequest, }; -use indexmap::IndexMap; const ROW_COUNTS: &[usize] = &[1_000, 10_000, 50_000, 100_000]; const COLUMN_COUNTS: &[usize] = &[8, 16, 32, 64]; @@ -171,40 +170,48 @@ fn build_dataset( dimensions: &[(String, String)], measures: &[(String, String)], time_dims: &[TimeColumn], -) -> JsRawData { +) -> JsRawColumnarData { let total_cols = dimensions.len() + measures.len() + time_dims.len(); - let mut rows = Vec::with_capacity(row_count); - - for i in 0..row_count { - let mut row = IndexMap::with_capacity(total_cols); - for (j, (_, alias)) in dimensions.iter().enumerate() { - row.insert( - alias.clone(), - DBResponsePrimitive::String(format!("dim_{}_{}", j, i % 1000)), - ); + let mut members = Vec::with_capacity(total_cols); + let mut columns: Vec> = Vec::with_capacity(total_cols); + + for (j, (_, alias)) in dimensions.iter().enumerate() { + members.push(alias.clone()); + let mut col = Vec::with_capacity(row_count); + for i in 0..row_count { + col.push(DBResponsePrimitive::String(format!( + "dim_{}_{}", + j, + i % 1000 + ))); } - for (j, (_, alias)) in measures.iter().enumerate() { - row.insert( - alias.clone(), - DBResponsePrimitive::Number(((i * (j + 1)) as f64) * 0.5), - ); + columns.push(col); + } + for (j, (_, alias)) in measures.iter().enumerate() { + members.push(alias.clone()); + let mut col = Vec::with_capacity(row_count); + for i in 0..row_count { + col.push(DBResponsePrimitive::Number(((i * (j + 1)) as f64) * 0.5)); } - for (j, td) in time_dims.iter().enumerate() { + columns.push(col); + } + for (j, td) in time_dims.iter().enumerate() { + members.push(td.alias.clone()); + let mut col = Vec::with_capacity(row_count); + for i in 0..row_count { // Format mirrors typical CubeStore output: ISO-8601 with millisecond - // fractional and no timezone. None of `transform_value`'s six chrono - // parsers fully match this shape, so the function falls through to - // `s.clone()` — measuring the production worst case. + // fractional and no timezone. let month = ((i + j) % 12) + 1; let day = ((i / 12) % 28) + 1; - row.insert( - td.alias.clone(), - DBResponsePrimitive::String(format!("2024-{:02}-{:02}T00:00:00.000", month, day)), - ); + col.push(DBResponsePrimitive::String(format!( + "2024-{:02}-{:02}T00:00:00.000", + month, day + ))); } - rows.push(row); + columns.push(col); } - rows + JsRawColumnarData { members, columns } } fn bench_transform(c: &mut Criterion) { @@ -305,5 +312,64 @@ fn bench_transform_time_scenarios(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_transform, bench_transform_time_scenarios); +/// Bench the JS→Rust raw-data ingest path: `serde_json::from_slice` then +/// `QueryResult::from_js_raw_data`. This is the part of the pipeline that the +/// columnar wire format change actually touches; `bench_transform` above +/// consumes an already-built `QueryResult` and is unaffected. +fn bench_from_js_raw_data(c: &mut Criterion) { + let mut group = c.benchmark_group("QueryResult::from_js_raw_data"); + + let combos: &[(usize, usize)] = &[(8, 10_000), (16, 10_000), (16, 100_000), (32, 100_000)]; + + for &(col_count, row_count) in combos { + let (dim_count, measure_count) = split_dim_measure(col_count); + let dimensions = make_member_aliases("dim", dim_count); + let measures = make_member_aliases("measure", measure_count); + + let dataset = build_dataset(row_count, &dimensions, &measures, &[]); + let payload = serde_json::to_vec(&dataset).expect("to_vec"); + let payload_len = payload.len(); + + eprintln!( + "from_js_raw_data: c{:02}_r{} payload_bytes={}", + col_count, row_count, payload_len + ); + + group.throughput(Throughput::Elements((row_count * col_count) as u64)); + + let id_param = format!("c{:02}_r{}", col_count, row_count); + + // Parse only: serde_json::from_slice into the wire type. + group.bench_with_input(BenchmarkId::new("parse_only", &id_param), &(), |b, _| { + b.iter(|| { + let parsed: JsRawColumnarData = + serde_json::from_slice(black_box(&payload)).expect("from_slice"); + black_box(parsed); + }); + }); + + // End-to-end: parse + transpose into QueryResult — what the Neon bridge does. + group.bench_with_input( + BenchmarkId::new("parse_plus_build", &id_param), + &(), + |b, _| { + b.iter(|| { + let parsed: JsRawColumnarData = + serde_json::from_slice(black_box(&payload)).expect("from_slice"); + let built = QueryResult::from_js_raw_data(parsed).expect("from_js_raw_data"); + black_box(built); + }); + }, + ); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_transform, + bench_transform_time_scenarios, + bench_from_js_raw_data +); criterion_main!(benches); diff --git a/rust/cube/cubeorchestrator/src/query_message_parser.rs b/rust/cube/cubeorchestrator/src/query_message_parser.rs index eb5f6c1cb6ac8..51ab4ac9c352e 100644 --- a/rust/cube/cubeorchestrator/src/query_message_parser.rs +++ b/rust/cube/cubeorchestrator/src/query_message_parser.rs @@ -1,6 +1,6 @@ use crate::{ query_result_transform::{DBResponsePrimitive, DBResponseValue}, - transport::JsRawData, + transport::JsRawColumnarData, }; use cubeshared::codegen::{root_as_http_message_with_opts, HttpCommand}; use cubeshared::flatbuffers::VerifierOptions; @@ -34,7 +34,7 @@ impl std::error::Error for ParseError {} #[derive(Debug, Clone)] pub struct QueryResult { - pub columns: Vec, + pub members: Vec, pub rows: Vec>, pub columns_pos: IndexMap, } @@ -44,7 +44,7 @@ impl Finalize for QueryResult {} impl QueryResult { pub fn from_cubestore_fb(msg_data: &[u8]) -> Result { let mut result = QueryResult { - columns: vec![], + members: vec![], rows: vec![], columns_pos: IndexMap::new(), }; @@ -76,7 +76,7 @@ impl QueryResult { return Err(ParseError::ColumnNameNotDefined); } - let (columns, columns_pos): (Vec<_>, IndexMap<_, _>) = result_set_columns + let (members, columns_pos): (Vec<_>, IndexMap<_, _>) = result_set_columns .iter() .enumerate() .map(|(index, column_name)| { @@ -84,7 +84,7 @@ impl QueryResult { }) .unzip(); - result.columns = columns; + result.members = members; result.columns_pos = columns_pos; } @@ -113,40 +113,41 @@ impl QueryResult { } } - pub fn from_js_raw_data(js_raw_data: JsRawData) -> Result { - if js_raw_data.is_empty() { + pub fn from_js_raw_data(js_raw_data: JsRawColumnarData) -> Result { + let JsRawColumnarData { members, columns } = js_raw_data; + + if members.is_empty() { return Ok(QueryResult { - columns: vec![], + members: vec![], rows: vec![], columns_pos: IndexMap::new(), }); } - let first_row = &js_raw_data[0]; - let columns: Vec = first_row.keys().cloned().collect(); - let columns_pos: IndexMap = columns + let columns_pos: IndexMap = members .iter() .enumerate() - .map(|(index, column)| (column.clone(), index)) + .map(|(index, member)| (member.clone(), index)) .collect(); - let rows: Vec> = js_raw_data - .into_iter() - .map(|row_map| { - columns - .iter() - .map(|col| { - row_map - .get(col) - .map(|val| DBResponseValue::Primitive(val.clone())) - .unwrap_or(DBResponseValue::Primitive(DBResponsePrimitive::Null)) - }) - .collect() - }) + let row_count = columns.first().map(|c| c.len()).unwrap_or(0); + // Transpose column-major input into the row-major shape `QueryResult` + // expects. Rows are pre-allocated, then we drain each column into the + // matching slot to avoid per-cell clones. + let mut rows: Vec> = (0..row_count) + .map(|_| Vec::with_capacity(members.len())) .collect(); + for column in columns.into_iter() { + for (row_idx, value) in column.into_iter().enumerate() { + if let Some(row) = rows.get_mut(row_idx) { + row.push(DBResponseValue::Primitive(value)); + } + } + } + Ok(QueryResult { - columns, + members, rows, columns_pos, }) @@ -233,7 +234,7 @@ mod tests { assert!(result.is_ok()); let query_result = result.unwrap(); - assert_eq!(query_result.columns.len(), 5); + assert_eq!(query_result.members.len(), 5); assert_eq!(query_result.rows.len(), 10); } @@ -245,7 +246,7 @@ mod tests { assert!(result.is_ok()); let query_result = result.unwrap(); - assert_eq!(query_result.columns.len(), 20); + assert_eq!(query_result.members.len(), 20); assert_eq!(query_result.rows.len(), 1000); } @@ -258,7 +259,7 @@ mod tests { assert!(result.is_ok()); let query_result = result.unwrap(); - assert_eq!(query_result.columns.len(), 30); + assert_eq!(query_result.members.len(), 30); assert_eq!(query_result.rows.len(), 10_000); } @@ -270,7 +271,7 @@ mod tests { assert!(result.is_ok()); let query_result = result.unwrap(); - assert_eq!(query_result.columns.len(), 40); + assert_eq!(query_result.members.len(), 40); assert_eq!(query_result.rows.len(), 33_000); } @@ -282,7 +283,7 @@ mod tests { assert!(result.is_ok()); let query_result = result.unwrap(); - assert_eq!(query_result.columns.len(), 100); + assert_eq!(query_result.members.len(), 100); assert_eq!(query_result.rows.len(), 50_000); } diff --git a/rust/cube/cubeorchestrator/src/query_result_transform.rs b/rust/cube/cubeorchestrator/src/query_result_transform.rs index 83223b2100346..a30139132e742 100644 --- a/rust/cube/cubeorchestrator/src/query_result_transform.rs +++ b/rust/cube/cubeorchestrator/src/query_result_transform.rs @@ -233,8 +233,8 @@ pub fn get_members( ) -> Result<(MembersMap, Vec)> { let mut members_map: MembersMap = IndexMap::new(); // IndexMap maintains insertion order, ensuring deterministic column ordering. - // The order comes from db_data.columns which now preserves the database result order - // (since JsRawData uses IndexMap instead of HashMap). + // `db_data.columns` preserves the original database result order — for JS + // input it's `JsRawColumnarData::members` (a `Vec` of column names in order). // Not sure if it solves the original comment below. // Original Comment: // Hashmaps don't guarantee the order of the elements while iterating @@ -243,7 +243,7 @@ pub fn get_members( // in sync with the order of members in members list. let mut members_arr: Vec = vec![]; - if db_data.columns.is_empty() { + if db_data.members.is_empty() { validate_query_members_in_annotation(query, annotation)?; return Ok((members_map, members_arr)); } @@ -252,7 +252,7 @@ pub fn get_members( // There is no granularity type/class implementation in rust yet. let mut minimal_granularities: HashMap = HashMap::new(); - for column in db_data.columns.iter() { + for column in db_data.members.iter() { let member_name = alias_to_member_name_map .get(column) .context(format!("Member name not found for alias: '{}'", column))?; @@ -1017,7 +1017,7 @@ impl Display for DBResponseValue { #[cfg(test)] mod tests { use super::*; - use crate::transport::JsRawData; + use crate::transport::JsRawColumnarData; use anyhow::Result; use chrono::{TimeZone, Timelike, Utc}; use serde_json::from_str; @@ -1029,7 +1029,7 @@ mod tests { #[serde(rename_all = "camelCase")] struct TestData { request: TransformDataRequest, - query_result: JsRawData, + query_result: JsRawColumnarData, final_result_default: Option, final_result_compact: Option, } @@ -1075,16 +1075,22 @@ mod tests { }, "queryType": "regularQuery" }, - "queryResult": [ - { - "e_commerce_records_us2021__city": "Missouri City", - "e_commerce_records_us2021__avg_discount": "0.80000000000000000000" - }, - { - "e_commerce_records_us2021__city": "Abilene", - "e_commerce_records_us2021__avg_discount": "0.80000000000000000000" - } - ], + "queryResult": { + "members": [ + "e_commerce_records_us2021__city", + "e_commerce_records_us2021__avg_discount" + ], + "columns": [ + [ + "Missouri City", + "Abilene" + ], + [ + "0.80000000000000000000", + "0.80000000000000000000" + ] + ] + }, "finalResultDefault": [ { "ECommerceRecordsUs2021.city": "Missouri City", @@ -1151,16 +1157,22 @@ mod tests { }, "queryType": "regularQuery" }, - "queryResult": [ - { - "e_commerce_records_us2021__postal_code": "95823", - "e_commerce_records_us2021__avg_profit": "646.1258666666666667" - }, - { - "e_commerce_records_us2021__postal_code": "64055", - "e_commerce_records_us2021__avg_profit": "487.8315000000000000" - } - ], + "queryResult": { + "members": [ + "e_commerce_records_us2021__postal_code", + "e_commerce_records_us2021__avg_profit" + ], + "columns": [ + [ + "95823", + "64055" + ], + [ + "646.1258666666666667", + "487.8315000000000000" + ] + ] + }, "finalResultDefault": [ { "ECommerceRecordsUs2021.postalCode": "95823", @@ -1255,16 +1267,22 @@ mod tests { }, "queryType": "compareDateRangeQuery" }, - "queryResult": [ - { - "e_commerce_records_us2021__order_date_day": "2020-01-01T00:00:00.000", - "e_commerce_records_us2021__count": "10" - }, - { - "e_commerce_records_us2021__order_date_day": null, - "e_commerce_records_us2021__count": null - } - ], + "queryResult": { + "members": [ + "e_commerce_records_us2021__order_date_day", + "e_commerce_records_us2021__count" + ], + "columns": [ + [ + "2020-01-01T00:00:00.000", + null + ], + [ + "10", + null + ] + ] + }, "finalResultDefault": [ { "ECommerceRecordsUs2021.orderDate.day": "2020-01-01T00:00:00.000", @@ -1369,16 +1387,22 @@ mod tests { }, "queryType": "compareDateRangeQuery" }, - "queryResult": [ - { - "e_commerce_records_us2021__order_date_day": "2020-03-02T00:00:00.000", - "e_commerce_records_us2021__count": "11" - }, - { - "e_commerce_records_us2021__order_date_day": "2020-03-03T00:00:00.000", - "e_commerce_records_us2021__count": "7" - } - ], + "queryResult": { + "members": [ + "e_commerce_records_us2021__order_date_day", + "e_commerce_records_us2021__count" + ], + "columns": [ + [ + "2020-03-02T00:00:00.000", + "2020-03-03T00:00:00.000" + ], + [ + "11", + "7" + ] + ] + }, "finalResultDefault": [ { "ECommerceRecordsUs2021.orderDate.day": "2020-03-02T00:00:00.000", @@ -1475,16 +1499,22 @@ mod tests { }, "queryType": "blendingQuery" }, - "queryResult": [ - { - "e_commerce_records_us2021__order_date_month": "2020-01-01T00:00:00.000", - "e_commerce_records_us2021__avg_discount": "0.15638297872340425532" - }, - { - "e_commerce_records_us2021__order_date_month": "2020-02-01T00:00:00.000", - "e_commerce_records_us2021__avg_discount": "0.17573529411764705882" - } - ], + "queryResult": { + "members": [ + "e_commerce_records_us2021__order_date_month", + "e_commerce_records_us2021__avg_discount" + ], + "columns": [ + [ + "2020-01-01T00:00:00.000", + "2020-02-01T00:00:00.000" + ], + [ + "0.15638297872340425532", + "0.17573529411764705882" + ] + ] + }, "finalResultDefault": [ { "ECommerceRecordsUs2021.orderDate.month": "2020-01-01T00:00:00.000", @@ -1581,16 +1611,22 @@ mod tests { }, "queryType": "blendingQuery" }, - "queryResult": [ - { - "e_commerce_records_us2021__order_date_month": "2020-01-01T00:00:00.000", - "e_commerce_records_us2021__avg_discount": "0.28571428571428571429" - }, - { - "e_commerce_records_us2021__order_date_month": "2020-02-01T00:00:00.000", - "e_commerce_records_us2021__avg_discount": "0.21777777777777777778" - } - ], + "queryResult": { + "members": [ + "e_commerce_records_us2021__order_date_month", + "e_commerce_records_us2021__avg_discount" + ], + "columns": [ + [ + "2020-01-01T00:00:00.000", + "2020-02-01T00:00:00.000" + ], + [ + "0.28571428571428571429", + "0.21777777777777777778" + ] + ] + }, "finalResultDefault": [ { "ECommerceRecordsUs2021.orderDate.month": "2020-01-01T00:00:00.000", @@ -1701,18 +1737,27 @@ mod tests { }, "queryType": "blendingQuery" }, - "queryResult": [ - { - "e_commerce_records_us2021__order_date_month": "2020-01-01T00:00:00.000", - "e_commerce_records_us2021__order_date_week": "2019-12-30T00:00:00.000", - "e_commerce_records_us2021__avg_discount": "0.28571428571428571429" - }, - { - "e_commerce_records_us2021__order_date_month": "2020-02-01T00:00:00.000", - "e_commerce_records_us2021__order_date_week": "2020-01-27T00:00:00.000", - "e_commerce_records_us2021__avg_discount": "0.21777777777777777778" - } - ], + "queryResult": { + "members": [ + "e_commerce_records_us2021__order_date_month", + "e_commerce_records_us2021__order_date_week", + "e_commerce_records_us2021__avg_discount" + ], + "columns": [ + [ + "2020-01-01T00:00:00.000", + "2020-02-01T00:00:00.000" + ], + [ + "2019-12-30T00:00:00.000", + "2020-01-27T00:00:00.000" + ], + [ + "0.28571428571428571429", + "0.21777777777777777778" + ] + ] + }, "finalResultDefault": [ { "ECommerceRecordsUs2021.orderDate.month": "2020-01-01T00:00:00.000", @@ -2530,7 +2575,7 @@ mod tests { query_type, query, &QueryResult { - columns: vec![], + members: vec![], rows: vec![], columns_pos: IndexMap::new(), }, @@ -2572,7 +2617,7 @@ mod tests { query_type, query, &QueryResult { - columns: vec![], + members: vec![], rows: vec![], columns_pos: IndexMap::new(), }, @@ -2620,7 +2665,7 @@ mod tests { query_type, query, &QueryResult { - columns: vec![], + members: vec![], rows: vec![], columns_pos: IndexMap::new(), }, @@ -2684,7 +2729,7 @@ mod tests { query_type, query, &QueryResult { - columns: vec![], + members: vec![], rows: vec![], columns_pos: IndexMap::new(), }, @@ -2756,7 +2801,7 @@ mod tests { query_type, query, &QueryResult { - columns: vec![], + members: vec![], rows: vec![], columns_pos: IndexMap::new(), }, diff --git a/rust/cube/cubeorchestrator/src/transport.rs b/rust/cube/cubeorchestrator/src/transport.rs index a52eb1793a89f..f0d8e85423e95 100644 --- a/rust/cube/cubeorchestrator/src/transport.rs +++ b/rust/cube/cubeorchestrator/src/transport.rs @@ -329,7 +329,13 @@ pub struct TransformDataRequest { pub res_type: Option, } -pub type JsRawData = Vec>; +/// Columnar representation of raw query results sent from JS to Rust. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct JsRawColumnarData { + pub members: Vec, + pub columns: Vec>, +} #[cfg(test)] mod tests { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/mod.rs index a84bb2d74c880..7f0a2f8f26e0b 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/mod.rs @@ -9,6 +9,7 @@ pub mod order; pub mod query_plan; pub mod schema; pub mod select; +pub mod symbols; pub mod time_series; pub mod union; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/case_dimension.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/case_dimension.rs new file mode 100644 index 0000000000000..ea7ed7d15af47 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/case_dimension.rs @@ -0,0 +1,16 @@ +use super::super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::symbols::dimension_kinds::CaseDimension; +use cubenativeutils::CubeError; + +impl ToSql for CaseDimension { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + if let Some(member_sql) = self.member_sql() { + ctx.eval_sql_call(member_sql) + } else { + Err(CubeError::internal(format!( + "Dimension {} has no sql evaluator", + ctx.full_name + ))) + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/mod.rs new file mode 100644 index 0000000000000..2a2e88261eedd --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/mod.rs @@ -0,0 +1,21 @@ +pub mod case_dimension; +pub mod regular; +pub mod switch; + +use super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::DimensionKind; +use cubenativeutils::CubeError; + +impl ToSql for DimensionKind { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + match self { + Self::Regular(r) => r.to_sql(ctx), + Self::Geo(_) => Err(CubeError::internal(format!( + "Geo dimension {} doesn't support evaluate_sql directly", + ctx.full_name + ))), + Self::Switch(s) => s.to_sql(ctx), + Self::Case(c) => c.to_sql(ctx), + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/regular.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/regular.rs new file mode 100644 index 0000000000000..d7422bfe881f5 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/regular.rs @@ -0,0 +1,9 @@ +use super::super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::symbols::dimension_kinds::RegularDimension; +use cubenativeutils::CubeError; + +impl ToSql for RegularDimension { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + ctx.eval_sql_call(self.member_sql()) + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/switch.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/switch.rs new file mode 100644 index 0000000000000..83cd4d84cdcef --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_kinds/switch.rs @@ -0,0 +1,13 @@ +use super::super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::symbols::dimension_kinds::SwitchDimension; +use cubenativeutils::CubeError; + +impl ToSql for SwitchDimension { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + if let Some(member_sql) = self.member_sql() { + ctx.eval_sql_call(member_sql) + } else { + ctx.templates.quote_identifier(ctx.name) + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_symbol.rs new file mode 100644 index 0000000000000..3ccb63e35cc35 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/dimension_symbol.rs @@ -0,0 +1,9 @@ +use super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::DimensionSymbol; +use cubenativeutils::CubeError; + +impl ToSql for DimensionSymbol { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + self.kind().to_sql(ctx) + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/aggregated.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/aggregated.rs new file mode 100644 index 0000000000000..8905ccdedb9bc --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/aggregated.rs @@ -0,0 +1,14 @@ +use super::super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::symbols::measure_kinds::AggregatedMeasure; +use cubenativeutils::CubeError; + +impl ToSql for AggregatedMeasure { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + match self.member_sql() { + Some(sql) => ctx.eval_sql_call(sql), + None => Err(CubeError::internal( + "Aggregated measure without sql cannot be evaluated directly".to_string(), + )), + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/calculated.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/calculated.rs new file mode 100644 index 0000000000000..112f3ec543507 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/calculated.rs @@ -0,0 +1,14 @@ +use super::super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::symbols::measure_kinds::CalculatedMeasure; +use cubenativeutils::CubeError; + +impl ToSql for CalculatedMeasure { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + match self.member_sql() { + Some(sql) => ctx.eval_sql_call(sql), + None => Err(CubeError::internal( + "Calculated measure without sql cannot be evaluated directly".to_string(), + )), + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/count.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/count.rs new file mode 100644 index 0000000000000..b7047bc20fb2f --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/count.rs @@ -0,0 +1,27 @@ +use super::super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::symbols::measure_kinds::{CountMeasure, CountSql}; +use cubenativeutils::CubeError; + +impl ToSql for CountMeasure { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + match self.sql() { + CountSql::Explicit(sql) => ctx.eval_sql_call(sql), + CountSql::Auto(pk_sqls) => { + if pk_sqls.len() > 1 { + let pk_strings = pk_sqls + .iter() + .map(|pk| -> Result<_, CubeError> { + let res = ctx.eval_sql_call(pk)?; + ctx.templates.cast_to_string(&res) + }) + .collect::, _>>()?; + ctx.templates.concat_strings(&pk_strings) + } else if let Some(pk_sql) = pk_sqls.first() { + ctx.eval_sql_call(pk_sql) + } else { + Ok("*".to_string()) + } + } + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/mod.rs new file mode 100644 index 0000000000000..b0bcc98ff39aa --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_kinds/mod.rs @@ -0,0 +1,21 @@ +pub mod aggregated; +pub mod calculated; +pub mod count; + +use super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::MeasureKind; +use cubenativeutils::CubeError; + +impl ToSql for MeasureKind { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + match self { + Self::Count(c) => c.to_sql(ctx), + Self::Aggregated(a) => a.to_sql(ctx), + Self::Calculated(c) => c.to_sql(ctx), + Self::Rank => Err(CubeError::internal(format!( + "Rank measure doesn't support direct evaluation for {}", + ctx.full_name + ))), + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_symbol.rs new file mode 100644 index 0000000000000..fc5074cc3d68b --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/measure_symbol.rs @@ -0,0 +1,9 @@ +use super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::MeasureSymbol; +use cubenativeutils::CubeError; + +impl ToSql for MeasureSymbol { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + self.kind().to_sql(ctx) + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/member_expression_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/member_expression_symbol.rs new file mode 100644 index 0000000000000..d8291e9a60539 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/member_expression_symbol.rs @@ -0,0 +1,20 @@ +use super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::{MemberExpressionExpression, MemberExpressionSymbol}; +use cubenativeutils::CubeError; + +impl ToSql for MemberExpressionSymbol { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + let sql = match self.expression() { + MemberExpressionExpression::SqlCall(sql_call) => ctx.eval_sql_call(sql_call)?, + MemberExpressionExpression::PatchedSymbol(symbol) => { + ctx.visitor + .apply(symbol, ctx.node_processor.clone(), ctx.templates)? + } + }; + if self.is_parenthesized() { + Ok(format!("({})", sql)) + } else { + Ok(sql) + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/member_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/member_symbol.rs new file mode 100644 index 0000000000000..918f54302c605 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/member_symbol.rs @@ -0,0 +1,14 @@ +use super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::MemberSymbol; +use cubenativeutils::CubeError; + +impl ToSql for MemberSymbol { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + match self { + Self::Dimension(d) => d.to_sql(ctx), + Self::TimeDimension(t) => t.to_sql(ctx), + Self::Measure(m) => m.to_sql(ctx), + Self::MemberExpression(e) => e.to_sql(ctx), + } + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/mod.rs new file mode 100644 index 0000000000000..e2099d83461c5 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/mod.rs @@ -0,0 +1,10 @@ +pub mod dimension_kinds; +pub mod dimension_symbol; +pub mod measure_kinds; +pub mod measure_symbol; +pub mod member_expression_symbol; +pub mod member_symbol; +pub mod time_dimension_symbol; +pub mod to_sql; + +pub use to_sql::{MemberSqlContext, ToSql}; diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/time_dimension_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/time_dimension_symbol.rs new file mode 100644 index 0000000000000..ba4649057b07e --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/time_dimension_symbol.rs @@ -0,0 +1,14 @@ +use super::{MemberSqlContext, ToSql}; +use crate::planner::sql_evaluator::TimeDimensionSymbol; +use cubenativeutils::CubeError; + +impl ToSql for TimeDimensionSymbol { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result { + let visitor = ctx.visitor.with_ignore_tz_convert(); + visitor.apply( + &self.base_symbol(), + ctx.node_processor.clone(), + ctx.templates, + ) + } +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/to_sql.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/to_sql.rs new file mode 100644 index 0000000000000..affa08712dd8f --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/plan/symbols/to_sql.rs @@ -0,0 +1,30 @@ +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::sql_nodes::SqlNode; +use crate::planner::sql_evaluator::{SqlCall, SqlEvaluatorVisitor}; +use crate::planner::sql_templates::PlanSqlTemplates; +use cubenativeutils::CubeError; +use std::rc::Rc; + +pub struct MemberSqlContext<'a> { + pub visitor: &'a SqlEvaluatorVisitor, + pub node_processor: &'a Rc, + pub query_tools: &'a Rc, + pub templates: &'a PlanSqlTemplates, + pub name: &'a str, + pub full_name: &'a str, +} + +impl<'a> MemberSqlContext<'a> { + pub fn eval_sql_call(&self, sql_call: &Rc) -> Result { + sql_call.eval( + self.visitor, + self.node_processor.clone(), + self.query_tools.clone(), + self.templates, + ) + } +} + +pub trait ToSql { + fn to_sql(&self, ctx: &MemberSqlContext) -> Result; +} diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs index 95066c71e9523..910681df594d6 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/sql_nodes/evaluate_sql.rs @@ -1,4 +1,5 @@ use super::SqlNode; +use crate::plan::symbols::{MemberSqlContext, ToSql}; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::MemberSymbol; use crate::planner::sql_evaluator::SqlEvaluatorVisitor; @@ -24,38 +25,16 @@ impl SqlNode for EvaluateSqlNode { node_processor: Rc, templates: &PlanSqlTemplates, ) -> Result { - let res = match node.as_ref() { - MemberSymbol::Dimension(ev) => { - let res = ev.evaluate_sql( - visitor, - node_processor.clone(), - query_tools.clone(), - templates, - )?; - Ok(res) - } - MemberSymbol::TimeDimension(ev) => { - let visitor = visitor.with_ignore_tz_convert(); - let res = visitor.apply(&ev.base_symbol(), node_processor.clone(), templates)?; - Ok(res) - } - MemberSymbol::Measure(ev) => ev.evaluate_sql( - visitor, - node_processor.clone(), - query_tools.clone(), - templates, - ), - MemberSymbol::MemberExpression(e) => { - let res = e.evaluate_sql( - visitor, - node_processor.clone(), - query_tools.clone(), - templates, - )?; - Ok(res) - } - }?; - Ok(res) + let path = node.compiled_path(); + let ctx = MemberSqlContext { + visitor, + node_processor: &node_processor, + query_tools: &query_tools, + templates, + name: path.name(), + full_name: path.full_name(), + }; + node.as_ref().to_sql(&ctx) } fn as_any(self: Rc) -> Rc { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/case_dimension.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/case_dimension.rs index b84df2734a140..01de4e0f1f1df 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/case_dimension.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/case_dimension.rs @@ -1,8 +1,6 @@ use super::super::common::{Case, DimensionType}; use super::super::MemberSymbol; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -42,24 +40,6 @@ impl CaseDimension { } } - pub fn evaluate_sql( - &self, - full_name: &str, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - if let Some(member_sql) = &self.member_sql { - member_sql.eval(visitor, node_processor, query_tools, templates) - } else { - Err(CubeError::internal(format!( - "Dimension {} hasn't sql evaluator", - full_name - ))) - } - } - pub fn get_dependencies(&self) -> Vec> { let mut deps = vec![]; if let Some(member_sql) = &self.member_sql { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/mod.rs index 7ad346fcbff29..e6a8f142c1b76 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/mod.rs @@ -10,9 +10,7 @@ pub use switch::*; use super::common::DimensionType; use super::MemberSymbol; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -25,30 +23,6 @@ pub enum DimensionKind { } impl DimensionKind { - pub fn evaluate_sql( - &self, - name: &str, - full_name: &str, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - match self { - Self::Regular(r) => r.evaluate_sql(visitor, node_processor, query_tools, templates), - Self::Geo(_) => Err(CubeError::internal(format!( - "Geo dimension {} doesn't support evaluate_sql directly", - full_name - ))), - Self::Switch(s) => { - s.evaluate_sql(name, visitor, node_processor, query_tools, templates) - } - Self::Case(c) => { - c.evaluate_sql(full_name, visitor, node_processor, query_tools, templates) - } - } - } - pub fn get_dependencies(&self) -> Vec> { match self { Self::Regular(r) => r.get_dependencies(), diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/regular.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/regular.rs index c1a88920a70df..42091cf6d7061 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/regular.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/regular.rs @@ -1,8 +1,6 @@ use super::super::common::DimensionType; use super::super::MemberSymbol; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -28,17 +26,6 @@ impl RegularDimension { &self.member_sql } - pub fn evaluate_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - self.member_sql - .eval(visitor, node_processor, query_tools, templates) - } - pub fn get_dependencies(&self) -> Vec> { let mut deps = vec![]; self.member_sql.extract_symbol_deps(&mut deps); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/switch.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/switch.rs index a8a8e2ed98bd2..5c61ba487b6fb 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/switch.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_kinds/switch.rs @@ -1,7 +1,5 @@ use super::super::MemberSymbol; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -28,21 +26,6 @@ impl SwitchDimension { self.member_sql.is_none() } - pub fn evaluate_sql( - &self, - name: &str, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - if let Some(member_sql) = &self.member_sql { - member_sql.eval(visitor, node_processor, query_tools, templates) - } else { - Ok(templates.quote_identifier(name)?) - } - } - pub fn get_dependencies(&self) -> Vec> { let mut deps = vec![]; if let Some(member_sql) = &self.member_sql { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs index 73c38b23f284f..955596ee22686 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/dimension_symbol.rs @@ -8,11 +8,8 @@ use super::{DimensionType, MemberSymbol, SymbolFactory}; use crate::cube_bridge::dimension_definition::DimensionDefinition; use crate::cube_bridge::evaluator::CubeEvaluator; use crate::cube_bridge::member_sql::MemberSql; -use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::TimeDimensionSymbol; -use crate::planner::sql_evaluator::{ - sql_nodes::SqlNode, Compiler, CubeRef, SqlCall, SqlEvaluatorVisitor, -}; +use crate::planner::sql_evaluator::{Compiler, CubeRef, SqlCall}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::GranularityHelper; use crate::planner::SqlInterval; @@ -73,23 +70,6 @@ impl DimensionSymbol { }) } - pub fn evaluate_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - self.kind.evaluate_sql( - self.compiled_path.name(), - self.compiled_path.full_name(), - visitor, - node_processor, - query_tools, - templates, - ) - } - pub fn is_calc_group(&self) -> bool { self.kind.is_calc_group() } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/aggregated.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/aggregated.rs index 0a4593bd28567..1ebaaf5282c67 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/aggregated.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/aggregated.rs @@ -1,8 +1,6 @@ use super::super::super::MemberSymbol; use super::super::common::AggregationType; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -35,21 +33,6 @@ impl AggregatedMeasure { self.member_sql.as_ref() } - pub fn evaluate_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - match &self.member_sql { - Some(sql) => sql.eval(visitor, node_processor, query_tools, templates), - None => Err(CubeError::internal( - "Aggregated measure without sql cannot be evaluated directly".to_string(), - )), - } - } - pub fn get_dependencies(&self) -> Vec> { let mut deps = vec![]; if let Some(sql) = &self.member_sql { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/calculated.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/calculated.rs index 95e3b3b32dab8..71ef664ec19c4 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/calculated.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/calculated.rs @@ -1,7 +1,5 @@ use super::super::super::MemberSymbol; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -63,21 +61,6 @@ impl CalculatedMeasure { self.member_sql.as_ref() } - pub fn evaluate_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - match &self.member_sql { - Some(sql) => sql.eval(visitor, node_processor, query_tools, templates), - None => Err(CubeError::internal( - "Calculated measure without sql cannot be evaluated directly".to_string(), - )), - } - } - pub fn get_dependencies(&self) -> Vec> { let mut deps = vec![]; if let Some(sql) = &self.member_sql { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/count.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/count.rs index d43b9e9d9ca17..673315d0207b2 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/count.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/count.rs @@ -1,7 +1,5 @@ use super::super::MemberSymbol; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -25,40 +23,6 @@ impl CountMeasure { &self.sql } - pub fn evaluate_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - match &self.sql { - CountSql::Explicit(sql) => sql.eval(visitor, node_processor, query_tools, templates), - CountSql::Auto(pk_sqls) => { - if pk_sqls.len() > 1 { - let pk_strings = pk_sqls - .iter() - .map(|pk| -> Result<_, CubeError> { - let res = pk.eval( - visitor, - node_processor.clone(), - query_tools.clone(), - templates, - )?; - templates.cast_to_string(&res) - }) - .collect::, _>>()?; - templates.concat_strings(&pk_strings) - } else if pk_sqls.len() == 1 { - let pk_sql = pk_sqls.first().unwrap(); - pk_sql.eval(visitor, node_processor, query_tools, templates) - } else { - Ok("*".to_string()) - } - } - } - } - pub fn get_dependencies(&self) -> Vec> { let mut deps = vec![]; match &self.sql { diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/mod.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/mod.rs index c217f201f51af..9ec7c1c2026f3 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/mod.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_kinds/mod.rs @@ -8,9 +8,7 @@ pub use count::*; use super::common::AggregationType; use super::MemberSymbol; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::{sql_nodes::SqlNode, CubeRef, SqlCall, SqlEvaluatorVisitor}; -use crate::planner::sql_templates::PlanSqlTemplates; +use crate::planner::sql_evaluator::{CubeRef, SqlCall}; use cubenativeutils::CubeError; use std::rc::Rc; @@ -62,25 +60,6 @@ impl MeasureKind { } } - pub fn evaluate_sql( - &self, - full_name: &str, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - match self { - Self::Count(c) => c.evaluate_sql(visitor, node_processor, query_tools, templates), - Self::Aggregated(a) => a.evaluate_sql(visitor, node_processor, query_tools, templates), - Self::Calculated(c) => c.evaluate_sql(visitor, node_processor, query_tools, templates), - Self::Rank => Err(CubeError::internal(format!( - "Rank measure doesn't support direct evaluation for {}", - full_name - ))), - } - } - pub fn get_dependencies(&self) -> Vec> { match self { Self::Count(c) => c.get_dependencies(), diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs index 3bd7bc0160239..5bd4d45d7951e 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/measure_symbol.rs @@ -5,11 +5,8 @@ use super::{MemberSymbol, SymbolFactory}; use crate::cube_bridge::evaluator::CubeEvaluator; use crate::cube_bridge::measure_definition::{MeasureDefinition, RollingWindow}; use crate::cube_bridge::member_sql::MemberSql; -use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::collectors::find_owned_by_cube_child; -use crate::planner::sql_evaluator::{ - sql_nodes::SqlNode, Compiler, CubeRef, SqlCall, SqlEvaluatorVisitor, -}; +use crate::planner::sql_evaluator::{Compiler, CubeRef, SqlCall}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::planner::SqlInterval; use cubenativeutils::CubeError; @@ -266,22 +263,6 @@ impl MeasureSymbol { } } - pub fn evaluate_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - self.kind.evaluate_sql( - &self.full_name(), - visitor, - node_processor, - query_tools, - templates, - ) - } - pub fn apply_to_deps) -> Result, CubeError>>( &self, f: &F, diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs index 69b218bad843f..1045e27773341 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/sql_evaluator/symbols/member_expression_symbol.rs @@ -1,10 +1,7 @@ use super::common::CompiledMemberPath; use super::MemberSymbol; -use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::collectors::member_childs; -use crate::planner::sql_evaluator::{ - sql_nodes::SqlNode, CubeRef, CubeTableSymbol, SqlCall, SqlEvaluatorVisitor, -}; +use crate::planner::sql_evaluator::{CubeRef, CubeTableSymbol, SqlCall}; use crate::planner::sql_templates::PlanSqlTemplates; use crate::utils::debug::DebugSql; use cubenativeutils::CubeError; @@ -52,34 +49,20 @@ impl MemberExpressionSymbol { })) } - pub fn evaluate_sql( - &self, - visitor: &SqlEvaluatorVisitor, - node_processor: Rc, - query_tools: Rc, - templates: &PlanSqlTemplates, - ) -> Result { - let sql = match &self.expression { - MemberExpressionExpression::SqlCall(sql_call) => { - sql_call.eval(visitor, node_processor, query_tools, templates)? - } - MemberExpressionExpression::PatchedSymbol(symbol) => { - visitor.apply(symbol, node_processor, templates)? - } - }; - if self.parenthesized { - Ok(format!("({})", sql)) - } else { - Ok(sql) - } - } - pub fn with_parenthesized(self: &Rc) -> Rc { let mut result = self.as_ref().clone(); result.parenthesized = true; Rc::new(result) } + pub fn expression(&self) -> &MemberExpressionExpression { + &self.expression + } + + pub fn is_parenthesized(&self) -> bool { + self.parenthesized + } + pub fn compiled_path(&self) -> &CompiledMemberPath { &self.compiled_path }