chore(tesseract): Fix issue with FILTER_PARAMS and issue with multi-stage behavior of non multi-stage members

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -1855,6 +1855,7 @@ export class BaseQuery {
         'collectSubQueryDimensionsFor'
       )
     ), inlineWhereConditions);
+
     return `SELECT ${this.selectAllDimensionsAndMeasures(measures)} FROM ${
       query
     } ${this.baseWhere(filters.concat(inlineWhereConditions))}` +
@@ -4001,6 +4002,23 @@ export class BaseQuery {
     );
   }
 
+  filtersProxyForRust(usedFilters) {
+    const filters = this.extractFiltersAsTree(usedFilters || []);
+    const allFilters = filters.map(this.initFilter.bind(this));
+    return BaseQuery.filterProxyFromAllFilters(
+      allFilters,
+      this.cubeEvaluator,
+      this.paramAllocator.allocateParam.bind(this.paramAllocator),
+      this.newGroupFilter.bind(this),
+    );
+  }
+
+  filterGroupFunctionForRust(usedFilters) {
+    const filters = this.extractFiltersAsTree(usedFilters || []);
+    const allFilters = filters.map(this.initFilter.bind(this));
+    return this.filterGroupFunctionImpl(allFilters);
+  }
+
   static renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter, aliases) {
     if (!filter) {
       return BaseFilter.ALWAYS_TRUE;
@@ -4046,6 +4064,10 @@ export class BaseQuery {
 
   filterGroupFunction() {
     const { allFilters } = this;
+    return this.filterGroupFunctionImpl(allFilters);
+  }
+
+  filterGroupFunctionImpl(allFilters) {
     const allocateParam = this.paramAllocator.allocateParam.bind(this.paramAllocator);
     const newGroupFilter = this.newGroupFilter.bind(this);
     return (...filterParamArgs) => {

packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts

@@ -1,3 +1,4 @@
+import { getEnv } from '@cubejs-backend/shared';
 import { UserError } from '../../../src/compiler/UserError';
 import { PostgresQuery } from '../../../src/adapter/PostgresQuery';
 import { BigqueryQuery } from '../../../src/adapter/BigqueryQuery';
@@ -312,9 +313,12 @@ describe('SQL Generation', () => {
 
     cube('visitor_checkins', {
       sql: \`
-      select * from visitor_checkins WHERE
-      \${FILTER_PARAMS.visitor_checkins.created_at.filter('created_at')} AND
-      \${FILTER_GROUP(FILTER_PARAMS.visitor_checkins.created_at.filter("(created_at - INTERVAL '3 DAY')"), FILTER_PARAMS.visitor_checkins.source.filter('source'))}
+      select visitor_checkins.* from visitor_checkins left join visitors on visitor_checkins.visitor_id = visitors.id WHERE
+      \${FILTER_PARAMS.visitor_checkins.created_at.filter('visitor_checkins.created_at')} AND
+      \${FILTER_GROUP(FILTER_PARAMS.visitor_checkins.created_at.filter("(visitor_checkins.created_at - INTERVAL '3 DAY')"), FILTER_PARAMS.visitor_checkins.source.filter('visitor_checkins.source'))}
+      AND \${SECURITY_CONTEXT.source.filter('visitors.source')} AND
+      \${SECURITY_CONTEXT.sourceArray.filter(sourceArray => \`visitors.source in (\${sourceArray.join(',')})\`)}
+
       \`,
       sql_alias: \`vc\`,
 
@@ -567,6 +571,84 @@ describe('SQL Generation', () => {
         },
       }
     });
+
+    cube('rollingWindowDates', {
+      sql: \`
+        SELECT cast('2024-01-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-02-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-03-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-04-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-05-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-06-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-07-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-08-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-09-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-10-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-11-13' AS timestamp) as time UNION ALL
+        SELECT cast('2024-12-13' AS timestamp) as time
+      \`,
+
+      dimensions: {
+        time: {
+          type: 'time',
+          sql: 'time',
+          primaryKey: true
+        }
+      }
+    });
+
+    cube('rollingWindowTest', {
+      sql: \`
+SELECT 1 AS revenue,  cast('2024-01-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-02-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-03-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-04-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-05-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-06-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-07-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-08-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-09-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-10-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-11-01' AS timestamp) as time UNION ALL
+      SELECT 1 AS revenue,  cast('2024-12-01' AS timestamp) as time
+      \`,
+
+      dimensions: {
+        time: {
+          type: 'time',
+          sql: 'time',
+          primaryKey: true
+        }
+      },
+      measures: {
+        revenue: {
+          sql: 'revenue',
+          type: 'sum',
+          filters: [{
+            sql: \`\${rollingWindowDates.time} <= current_date\`
+          }]
+        },
+        revenue_ytd: {
+          sql: \`\${CUBE.revenue}\`,
+          type: 'sum',
+          rolling_window: {
+            type: 'to_date',
+            granularity: 'year'
+          }
+        },
+        revenue_ms: {
+          sql: \`\${CUBE.revenue}\`,
+          type: 'sum',
+          multi_stage: true,
+        },
+      },
+      joins: {
+        rollingWindowDates: {
+          relationship: 'manyToOne',
+          sql: \`\${CUBE}.time = date_trunc('month', \${rollingWindowDates.time})\`
+        }
+      }
+    });
     `);
 
   it('simple join', async () => {
@@ -1966,7 +2048,7 @@ describe('SQL Generation', () => {
   ]));
 
   it(
-    'contains filter 1',
+    'contains filter',
     () => runQueryTest({
       measures: [],
       dimensions: [
@@ -2761,7 +2843,7 @@ describe('SQL Generation', () => {
     ]
   ));
 
-  it('rank measure 1', async () => runQueryTest(
+  it('rank measure', async () => runQueryTest(
     {
       measures: ['visitors.revenue_rank'],
     },
@@ -3152,6 +3234,37 @@ describe('SQL Generation', () => {
     }]
   ));
 
+  if (getEnv('nativeSqlPlanner')) {
+    it('nested aggregations with filtered measures and rolling windows', async () => runQueryTest(
+      {
+        measures: ['rollingWindowTest.revenue_ms'],
+      },
+      [{
+        rolling_window_test__revenue_ms: '12'
+      }]
+    ));
+  }
+
+  it('multiplied sum and count no dimensions through view', async () => runQueryTest(
+    {
+      measures: ['visitors_visitors_checkins_view.revenue', 'visitors_visitors_checkins_view.visitor_checkins_count'],
+    },
+    [{
+      visitors_visitors_checkins_view__revenue: '2000',
+      visitors_visitors_checkins_view__visitor_checkins_count: '6'
+    }]
+  ));
+
+  it('multiplied sum no dimensions through view', async () => runQueryTest(
+    {
+      measures: ['visitors_visitors_checkins_view.revenue', 'visitors_visitors_checkins_view.id_sum'],
+    },
+    [{
+      visitors_visitors_checkins_view__revenue: '2000',
+      visitors_visitors_checkins_view__id_sum: '21'
+    }]
+  ));
+
   // Subquery aggregation for multiplied measure (and any `keysSelect` for that matter)
   // should pick up all dimensions, even through member expressions
   it('multiplied sum with dimension member expressions', async () => runQueryTest(

rust/cubenativeutils/src/wrappers/context.rs

@@ -6,6 +6,7 @@ pub trait NativeContext<IT: InnerTypes>: Clone {
     fn string(&self, v: String) -> Result<IT::String, CubeError>;
     fn number(&self, v: f64) -> Result<IT::Number, CubeError>;
     fn undefined(&self) -> Result<NativeObjectHandle<IT>, CubeError>;
+    fn null(&self) -> Result<NativeObjectHandle<IT>, CubeError>;
     fn empty_array(&self) -> Result<IT::Array, CubeError>;
     fn empty_struct(&self) -> Result<IT::Struct, CubeError>;
     //fn boxed<T: 'static>(&self, value: T) -> impl NativeBox<IT, T>;
@@ -36,6 +37,9 @@ impl<IT: InnerTypes> NativeContextHolder<IT> {
     pub fn undefined(&self) -> Result<NativeObjectHandle<IT>, CubeError> {
         self.context.undefined()
     }
+    pub fn null(&self) -> Result<NativeObjectHandle<IT>, CubeError> {
+        self.context.null()
+    }
     pub fn empty_array(&self) -> Result<IT::Array, CubeError> {
         self.context.empty_array()
     }

rust/cubenativeutils/src/wrappers/neon/context.rs

@@ -144,6 +144,13 @@ impl<C: Context<'static> + 'static> NativeContext<NeonInnerTypes<C>> for Context
         )))
     }
 
+    fn null(&self) -> Result<NativeObjectHandle<NeonInnerTypes<C>>, CubeError> {
+        Ok(NativeObjectHandle::new(NeonObject::new(
+            self.clone(),
+            self.with_context(|cx| cx.null().upcast())?,
+        )))
+    }
+
     fn empty_array(&self) -> Result<NeonArray<C>, CubeError> {
         let obj = NeonObject::new(
             self.clone(),

rust/cubenativeutils/src/wrappers/serializer/serializer.rs

@@ -138,7 +138,7 @@ impl<IT: InnerTypes> ser::Serializer for NativeSerdeSerializer<IT> {
     }
 
     fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
-        Ok(self.context.undefined()?)
+        Ok(self.context.null()?)
     }
 
     fn serialize_some<T>(self, value: &T) -> Result<Self::Ok, Self::Error>

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_query_options.rs

@@ -23,7 +23,7 @@ pub struct TimeDimension {
 pub struct FilterItem {
     pub or: Option<Vec<FilterItem>>,
     pub and: Option<Vec<FilterItem>>,
-    member: Option<String>,
+    pub member: Option<String>,
     pub dimension: Option<String>,
     pub operator: Option<String>,
     pub values: Option<Vec<Option<String>>>,

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/base_tools.rs

@@ -1,3 +1,4 @@
+use super::base_query_options::FilterItem;
 use super::filter_group::{FilterGroup, NativeFilterGroup};
 use super::filter_params::{FilterParams, NativeFilterParams};
 use super::member_sql::{MemberSql, NativeMemberSql};
@@ -36,8 +37,14 @@ pub trait BaseTools {
     ) -> Result<Vec<CallDep>, CubeError>;
     fn security_context_for_rust(&self) -> Result<Rc<dyn SecurityContext>, CubeError>;
     fn sql_utils_for_rust(&self) -> Result<Rc<dyn SqlUtils>, CubeError>;
-    fn filters_proxy(&self) -> Result<Rc<dyn FilterParams>, CubeError>;
-    fn filter_group_function(&self) -> Result<Rc<dyn FilterGroup>, CubeError>;
+    fn filters_proxy_for_rust(
+        &self,
+        used_filters: Option<Vec<FilterItem>>,
+    ) -> Result<Rc<dyn FilterParams>, CubeError>;
+    fn filter_group_function_for_rust(
+        &self,
+        used_filters: Option<Vec<FilterItem>>,
+    ) -> Result<Rc<dyn FilterGroup>, CubeError>;
     fn timestamp_precision(&self) -> Result<u32, CubeError>;
     fn in_db_time_zone(&self, date: String) -> Result<String, CubeError>;
     fn generate_time_series(

rust/cubesqlplanner/cubesqlplanner/src/plan/builder/select.rs

@@ -193,11 +193,11 @@ impl SelectBuilder {
         Select {
             projection_columns: self.projection_columns,
             from: self.from,
-            filter: self.filter,
+            filter: self.filter.clone(),
             group_by: self.group_by,
             having: self.having,
             order_by: self.order_by,
-            context: Rc::new(VisitorContext::new(&nodes_factory)),
+            context: Rc::new(VisitorContext::new(&nodes_factory, self.filter)),
             ctes: self.ctes,
             is_distinct: self.is_distinct,
             limit: self.limit,

rust/cubesqlplanner/cubesqlplanner/src/planner/filter/compiler.rs

@@ -129,7 +129,7 @@ impl<'a> FilterCompiler<'a> {
             } else {
                 Err(CubeError::user(format!(
                     "Member and operator attributes is required for filter"
-                ))) //TODO pring condition
+                ))) //TODO print condition
             }
         }
     }

rust/cubesqlplanner/cubesqlplanner/src/planner/filter/filter_operator.rs

@@ -53,3 +53,31 @@ impl FromStr for FilterOperator {
         }
     }
 }
+
+impl ToString for FilterOperator {
+    fn to_string(&self) -> String {
+        let str = match self {
+            FilterOperator::Equal => "equals",
+            FilterOperator::NotEqual => "notEquals",
+            FilterOperator::InDateRange => "inDateRange",
+            FilterOperator::RegularRollingWindowDateRange => "inDateRange",
+            FilterOperator::ToDateRollingWindowDateRange => "inDateRange",
+            FilterOperator::In => "in",
+            FilterOperator::NotIn => "notIn",
+            FilterOperator::Set => "set",
+            FilterOperator::NotSet => "notSet",
+            FilterOperator::Gt => "gt",
+            FilterOperator::Gte => "gte",
+            FilterOperator::Lt => "lt",
+            FilterOperator::Lte => "lte",
+            FilterOperator::Contains => "contains",
+            FilterOperator::NotContains => "notContains",
+            FilterOperator::StartsWith => "startsWith",
+            FilterOperator::NotStartsWith => "notStartsWith",
+            FilterOperator::NotEndsWith => "notEndsWith",
+            FilterOperator::EndsWith => "endsWith",
+            FilterOperator::MeasureFilter => "measureFilter",
+        };
+        str.to_string()
+    }
+}

rust/cubesqlplanner/cubesqlplanner/src/planner/planners/multi_stage/applied_state.rs

@@ -173,6 +173,7 @@ impl MultiStageAppliedState {
             &self.time_dimensions_filters,
             &operator,
             &values,
+            &None,
         );
     }
 
@@ -188,6 +189,24 @@ impl MultiStageAppliedState {
             &self.time_dimensions_filters,
             &operator,
             &values,
+            &None,
+        );
+    }
+
+    pub fn replace_range_in_date_filter(
+        &mut self,
+        member_name: &String,
+        new_from: String,
+        new_to: String,
+    ) {
+        let operator = FilterOperator::InDateRange;
+        let replacement_values = vec![Some(new_from), Some(new_to)];
+        self.time_dimensions_filters = self.change_date_range_filter_impl(
+            member_name,
+            &self.time_dimensions_filters,
+            &operator,
+            &vec![],
+            &Some(replacement_values),
         );
     }
 
@@ -197,6 +216,7 @@ impl MultiStageAppliedState {
         filters: &Vec<FilterItem>,
         operator: &FilterOperator,
         additional_values: &Vec<Option<String>>,
+        replacement_values: &Option<Vec<Option<String>>>,
     ) -> Vec<FilterItem> {
         let mut result = Vec::new();
         for item in filters.iter() {
@@ -209,6 +229,7 @@ impl MultiStageAppliedState {
                             filters,
                             operator,
                             additional_values,
+                            replacement_values,
                         ),
                     )));
                     result.push(new_group);
@@ -217,7 +238,11 @@ impl MultiStageAppliedState {
                     let itm = if &itm.member_name() == member_name
                         && matches!(itm.filter_operator(), FilterOperator::InDateRange)
                     {
-                        let mut values = itm.values().clone();
+                        let mut values = if let Some(values) = replacement_values {
+                            values.clone()
+                        } else {
+                            itm.values().clone()
+                        };
                         values.extend(additional_values.iter().cloned());
                         itm.change_operator(operator.clone(), values)
                     } else {