From 53477e460f554d06fe84237fd45e9e0a0cf35cfd Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:04:17 +0000 Subject: [PATCH 1/3] refactor: remove the legacy frontend modules replaced by the unified ones Since the previous commit nothing calls the legacy SQL, PromQL and MetricsQL lowerings or their tests; the unified modules take their paths in the next commit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/frontend-metricsql/src/lib.rs | 329 -- crates/frontend-promql/src/error.rs | 81 - crates/frontend-promql/src/histogram.rs | 137 - crates/frontend-promql/src/lib.rs | 196 -- crates/frontend-promql/src/promql.rs | 2169 ------------- .../tests/histogram_metadata.rs | 140 - .../tests/promql_conformance.rs | 2291 -------------- .../frontend-promql/tests/promql_lowering.rs | 1346 -------- crates/frontend-promql/tests/support.rs | 52 - crates/frontend-sql/src/error.rs | 75 - crates/frontend-sql/src/lib.rs | 107 - crates/frontend-sql/src/sql/clickhouse_ast.rs | 152 - .../src/sql/collection_planning.rs | 194 -- crates/frontend-sql/src/sql/expr.rs | 328 -- crates/frontend-sql/src/sql/mod.rs | 2523 --------------- crates/frontend-sql/src/sql/types.rs | 378 --- crates/frontend-sql/tests/sql_lowering.rs | 2718 ----------------- 17 files changed, 13216 deletions(-) delete mode 100644 crates/frontend-metricsql/src/lib.rs delete mode 100644 crates/frontend-promql/src/error.rs delete mode 100644 crates/frontend-promql/src/histogram.rs delete mode 100644 crates/frontend-promql/src/lib.rs delete mode 100644 crates/frontend-promql/src/promql.rs delete mode 100644 crates/frontend-promql/tests/histogram_metadata.rs delete mode 100644 crates/frontend-promql/tests/promql_conformance.rs delete mode 100644 crates/frontend-promql/tests/promql_lowering.rs delete mode 100644 crates/frontend-promql/tests/support.rs delete mode 100644 crates/frontend-sql/src/error.rs delete mode 100644 crates/frontend-sql/src/lib.rs delete mode 100644 crates/frontend-sql/src/sql/clickhouse_ast.rs delete mode 100644 crates/frontend-sql/src/sql/collection_planning.rs delete mode 100644 crates/frontend-sql/src/sql/expr.rs delete mode 100644 crates/frontend-sql/src/sql/mod.rs delete mode 100644 crates/frontend-sql/src/sql/types.rs delete mode 100644 crates/frontend-sql/tests/sql_lowering.rs diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs deleted file mode 100644 index 4417a0be3..000000000 --- a/crates/frontend-metricsql/src/lib.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! MetricsQL AST to canonical `QueryExpr` frontend. - -use std::{rc::Rc, time::Duration}; - -use asap_types::pre_asap::{ - resolve_root, AggIntent, ArithmeticOpKind, BinaryOpKind, ColumnRef, CompareOpKind, GroupKeys, - Predicate, PromQLVectorSetOpKind, QueryExpr, Reduction, ScalarValue, Source, - UnresolvedQueryExpr as U, -}; -use asap_types::types::AccuracyTarget; -use metricsql_parser::ast::{AggregateModifier, DurationExpr, Expr, MetricExpr, RollupExpr}; -use metricsql_parser::functions::{AggregateFunction, BuiltinFunction, RollupFunction}; -use metricsql_parser::label::{LabelFilter, LabelFilterOp, NAME_LABEL}; -use thiserror::Error; - -pub use metricsql_parser::ast::Expr as MetricsqlExpr; - -#[derive(Debug, Error)] -pub enum MetricsqlError { - #[error("MetricsQL parse error: {0}")] - Parse(String), - #[error("unsupported MetricsQL feature: {0}")] - UnsupportedFeature(String), - #[error("MetricsQL column resolution failed: {0}")] - Resolve(String), -} - -pub fn parse_metricsql(query: &str) -> Result { - metricsql_parser::parser::parse(query).map_err(|e| MetricsqlError::Parse(e.to_string())) -} - -pub fn canonical_metricsql(query: &str) -> Result { - Ok(parse_metricsql(query)?.to_string()) -} - -pub fn lower_metricsql(query: &str, accuracy: AccuracyTarget) -> Result { - let ast = parse_metricsql(query)?; - let unresolved = Lowerer { accuracy }.lower(&ast)?; - resolve_root(&unresolved).map_err(|e| MetricsqlError::Resolve(e.to_string())) -} - -struct Lowerer { - accuracy: AccuracyTarget, -} - -impl Lowerer { - fn lower(&self, expr: &Expr) -> Result { - match expr { - Expr::MetricExpression(e) => self.metric(e), - Expr::Rollup(e) => self.rollup(e), - Expr::Function(e) => self.function(e), - Expr::Aggregation(e) => self.aggregate(e), - Expr::NumberLiteral(e) => Ok(U::promql_scalar(e.value)), - Expr::UnaryOperator(e) => Ok(U::BinaryOp { - op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - lhs: Rc::new(self.lower(&e.expr)?), - rhs: Rc::new(U::promql_scalar(-1.0)), - vector_match: None, - }), - Expr::BinaryOperator(e) => self.binary(e), - Expr::Parens(e) if e.expressions.len() == 1 => self.lower(&e.expressions[0]), - Expr::With(e) => self.lower(&e.expr), - other => Err(unsupported(format!("AST node `{other}`"))), - } - } - - fn metric(&self, metric: &MetricExpr) -> Result { - if metric.has_or_matchers() { - return Err(unsupported("or-delimited selector matchers")); - } - let name = metric - .metric_name() - .ok_or_else(|| unsupported("selector without one exact metric name"))?; - let mut filters: Vec<_> = metric - .matchers - .filter_iter() - .filter(|f| f.label != NAME_LABEL) - .collect(); - filters.sort_by(|a, b| a.label.cmp(&b.label).then(a.value.cmp(&b.value))); - Ok(U::Scan { - source: Source::TimeSeries { - metric: name.to_owned(), - }, - predicates: filters - .into_iter() - .map(|f| Predicate(Rc::new(matcher(f)))) - .collect(), - schema: None, - }) - } - - fn rollup(&self, rollup: &RollupExpr) -> Result { - if rollup.offset.is_some() || rollup.at.is_some() { - return Err(unsupported("offset and @ modifiers")); - } - if rollup.for_subquery() { - return Err(unsupported("subquery step or inherited step")); - } - let child = self.lower(&rollup.expr)?; - match &rollup.window { - None => Ok(child), - Some(window) => Ok(U::TimeRange { - range: duration(window)?, - child: Rc::new(child), - }), - } - } - - fn function( - &self, - function: &metricsql_parser::ast::FunctionExpr, - ) -> Result { - if function.keep_metric_names { - return Err(unsupported( - "keep_metric_names requires metric-name lineage", - )); - } - let BuiltinFunction::Rollup(rollup) = function.function else { - return Err(unsupported(format!("function `{}`", function.name()))); - }; - let expected_args = if rollup == RollupFunction::QuantileOverTime { - 2 - } else { - 1 - }; - require_arity(function.name(), function.args.len(), expected_args)?; - let child_index = usize::from(rollup == RollupFunction::QuantileOverTime); - let child = function - .args - .get(child_index) - .ok_or_else(|| unsupported(format!("missing argument for `{}`", function.name())))?; - let intent = match rollup { - RollupFunction::DefaultRollup | RollupFunction::LastOverTime => AggIntent::LastOverTime, - RollupFunction::FirstOverTime => AggIntent::FirstOverTime, - RollupFunction::AvgOverTime => AggIntent::Avg { col: None }, - RollupFunction::MinOverTime => AggIntent::Min { col: None }, - RollupFunction::MaxOverTime => AggIntent::Max { col: None }, - RollupFunction::SumOverTime => AggIntent::Sum { col: None }, - RollupFunction::CountOverTime => AggIntent::Count { - accuracy: self.accuracy.clone(), - }, - RollupFunction::StddevOverTime => AggIntent::StdDev { - col: None, - population: true, - }, - RollupFunction::StdvarOverTime => AggIntent::Variance { - col: None, - population: true, - }, - RollupFunction::Rate => AggIntent::Rate, - RollupFunction::IRate => AggIntent::IRate, - RollupFunction::Increase => AggIntent::Increase, - RollupFunction::Changes => AggIntent::Changes, - RollupFunction::Delta => AggIntent::Delta, - RollupFunction::IDelta => AggIntent::IDelta, - RollupFunction::Deriv => AggIntent::Deriv, - RollupFunction::Resets => AggIntent::Resets, - RollupFunction::MadOverTime => AggIntent::MadOverTime, - RollupFunction::PresentOverTime => AggIntent::PresentOverTime, - RollupFunction::AbsentOverTime => AggIntent::AbsentOverTime, - RollupFunction::QuantileOverTime => AggIntent::Quantile { - col: None, - q: number_arg(&function.args, 0)?, - accuracy: self.accuracy.clone(), - }, - _ => { - return Err(unsupported(format!( - "rollup function `{}`", - function.name() - ))) - } - }; - let child = self.lower(child)?; - if rollup == RollupFunction::DefaultRollup && !matches!(child, U::TimeRange { .. }) { - return Err(unsupported( - "default_rollup without an explicit range requires an evaluation step", - )); - } - Ok(aggregate(Reduction::PerEntity, intent, child)) - } - - fn aggregate( - &self, - expr: &metricsql_parser::ast::AggregationExpr, - ) -> Result { - if expr.limit != 0 || expr.keep_metric_names { - return Err(unsupported("aggregate limit or keep_metric_names")); - } - let expected_args = if expr.function == AggregateFunction::Quantile { - 2 - } else { - 1 - }; - require_arity(expr.name(), expr.args.len(), expected_args)?; - let child_index = expr - .arg_idx_for_optimization() - .ok_or_else(|| unsupported(format!("aggregate `{}` arguments", expr.name())))?; - let intent = match expr.function { - AggregateFunction::Sum => AggIntent::Sum { col: None }, - AggregateFunction::Avg => AggIntent::Avg { col: None }, - AggregateFunction::Min => AggIntent::Min { col: None }, - AggregateFunction::Max => AggIntent::Max { col: None }, - AggregateFunction::Count => AggIntent::Cardinality { - cols: vec![], - accuracy: self.accuracy.clone(), - }, - AggregateFunction::StdDev => AggIntent::StdDev { - col: None, - population: true, - }, - AggregateFunction::StdVar => AggIntent::Variance { - col: None, - population: true, - }, - AggregateFunction::Group => AggIntent::Group, - AggregateFunction::Quantile => AggIntent::Quantile { - col: None, - q: number_arg(&expr.args, 0)?, - accuracy: self.accuracy.clone(), - }, - _ => return Err(unsupported(format!("aggregate `{}`", expr.name()))), - }; - let reduction = match &expr.modifier { - None => Reduction::by(vec![]), - Some(AggregateModifier::By(v)) => Reduction::by(names(v)), - Some(AggregateModifier::Without(v)) => Reduction::Reduce(GroupKeys::without(names(v))), - }; - let child = expr - .args - .get(child_index) - .ok_or_else(|| unsupported("missing aggregate input"))?; - Ok(aggregate(reduction, intent, self.lower(child)?)) - } - - fn binary(&self, expr: &metricsql_parser::ast::BinaryExpr) -> Result { - if expr.modifier.is_some() { - return Err(unsupported("binary vector matching modifiers")); - } - use metricsql_parser::ast::Operator as O; - let op = match expr.op { - O::Add => BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - O::Sub => BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub), - O::Mul => BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - O::Div => BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), - O::Mod => BinaryOpKind::Arithmetic(ArithmeticOpKind::Mod), - O::Pow => BinaryOpKind::Arithmetic(ArithmeticOpKind::Pow), - O::Atan2 => BinaryOpKind::Arithmetic(ArithmeticOpKind::Atan2), - O::Eql => BinaryOpKind::Compare(CompareOpKind::Eq), - O::NotEq => BinaryOpKind::Compare(CompareOpKind::Ne), - O::Lt => BinaryOpKind::Compare(CompareOpKind::Lt), - O::Lte => BinaryOpKind::Compare(CompareOpKind::Le), - O::Gt => BinaryOpKind::Compare(CompareOpKind::Gt), - O::Gte => BinaryOpKind::Compare(CompareOpKind::Ge), - O::And => BinaryOpKind::Set(PromQLVectorSetOpKind::And), - O::Or => BinaryOpKind::Set(PromQLVectorSetOpKind::Or), - O::Unless => BinaryOpKind::Set(PromQLVectorSetOpKind::Unless), - O::If | O::IfNot | O::Default => { - return Err(unsupported(format!("MetricsQL operator `{}`", expr.op))) - } - }; - Ok(U::BinaryOp { - op, - lhs: Rc::new(self.lower(&expr.left)?), - rhs: Rc::new(self.lower(&expr.right)?), - vector_match: None, - }) - } -} - -fn names(values: &[String]) -> Vec { - values.iter().cloned().map(ColumnRef::Named).collect() -} - -fn aggregate(reduction: Reduction, intent: AggIntent, child: U) -> U { - U::Aggregate { - reduction, - measures: vec![intent], - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -fn matcher(filter: &LabelFilter) -> U { - let op = match filter.op { - LabelFilterOp::Equal => CompareOpKind::Eq, - LabelFilterOp::NotEqual => CompareOpKind::Ne, - LabelFilterOp::RegexEqual => CompareOpKind::Regex, - LabelFilterOp::RegexNotEqual => CompareOpKind::NotRegex, - }; - U::Compare { - left: Rc::new(U::Column(ColumnRef::Named(filter.label.clone()))), - op, - right: Rc::new(U::Literal(ScalarValue::Utf8(filter.value.clone()))), - } -} - -fn duration(value: &DurationExpr) -> Result { - match value { - DurationExpr::Millis(ms) if *ms >= 0 => Ok(Duration::from_millis(*ms as u64)), - DurationExpr::StepValue(_) => Err(unsupported("step-relative duration")), - DurationExpr::Millis(_) => Err(unsupported("negative duration")), - } -} - -fn number_arg(args: &[Expr], index: usize) -> Result { - match args.get(index) { - Some(Expr::NumberLiteral(v)) if v.value.is_finite() => Ok(v.value), - _ => Err(unsupported(format!("numeric argument #{index}"))), - } -} - -fn require_arity(name: &str, actual: usize, expected: usize) -> Result<(), MetricsqlError> { - if actual == expected { - Ok(()) - } else { - Err(unsupported(format!( - "`{name}` with {actual} arguments; canonical lowering requires exactly {expected}" - ))) - } -} - -fn unsupported(message: impl Into) -> MetricsqlError { - MetricsqlError::UnsupportedFeature(message.into()) -} - -/// Unified lowering, promoted to the root API at planner cutover. -pub mod unified; diff --git a/crates/frontend-promql/src/error.rs b/crates/frontend-promql/src/error.rs deleted file mode 100644 index 6f996b11d..000000000 --- a/crates/frontend-promql/src/error.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::fmt; - -use asap_types::pre_asap::ResolveDAGError; -use asap_types::workload::WorkloadError; - -/// Errors from lowering a PromQL query (parse → the canonical, unresolved -/// DAG, built directly → -/// [`resolve_root`](asap_types::pre_asap::resolve_root) binds it to the -/// resolved DAG, issue #179). -/// -/// Carries no DataFusion type — the PromQL front end never depends on the SQL -/// stack. The language-neutral variants (`UnsupportedFeature` / `WrongLanguage` -/// / `Convert`) are mirrored by [`asap_frontend_sql::SqlError`] rather than -/// shared, so neither front end pulls the other's parser. -#[derive(Debug)] -pub enum PromqlError { - /// The workload omitted information required for plan-ready PromQL lowering. - InvalidWorkload(WorkloadError), - /// The `promql-parser` crate rejected the query string (parse failure). - Parse(String), - /// A PromQL function (`rate`, `*_over_time`, …) not supported in this version. - UnsupportedFunction(String), - /// A PromQL aggregation operator (`sum`, `topk`, …) not supported. - UnsupportedAggregateOp(String), - /// A structural feature (offset / `@` / `without`) not supported in this - /// version. - UnsupportedFeature(String), - /// A required function / aggregator argument was missing. - MissingArgument(String), - /// An argument had the wrong shape (e.g. a non-numeric `topk` parameter). - InvalidParameter(String), - /// The workload's query language is not PromQL. - WrongLanguage(String), - /// Resolving the canonical unresolved DAG failed (name resolution - /// against the bound schema). - Convert(ResolveDAGError), -} - -impl fmt::Display for PromqlError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::InvalidWorkload(e) => write!(f, "invalid PromQL workload: {e}"), - Self::Parse(e) => write!(f, "PromQL parse error: {e}"), - Self::UnsupportedFunction(n) => write!(f, "unsupported PromQL function: {n}"), - Self::UnsupportedAggregateOp(n) => write!(f, "unsupported PromQL aggregate op: {n}"), - Self::UnsupportedFeature(m) => write!(f, "unsupported feature: {m}"), - Self::MissingArgument(m) => write!(f, "missing argument: {m}"), - Self::InvalidParameter(m) => write!(f, "invalid parameter: {m}"), - Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), - Self::Convert(e) => write!(f, "column resolution failed: {e}"), - } - } -} - -impl std::error::Error for PromqlError {} - -impl From for PromqlError { - fn from(e: ResolveDAGError) -> Self { - Self::Convert(e) - } -} - -impl From for PromqlError { - fn from(e: WorkloadError) -> Self { - Self::InvalidWorkload(e) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn unsupported_feature_label_is_language_neutral() { - // `UnsupportedFeature` shares a Display label with the SQL side, so it - // must not hardcode "PromQL". - let msg = PromqlError::UnsupportedFeature("subquery".into()).to_string(); - assert_eq!(msg, "unsupported feature: subquery"); - assert!(!msg.contains("PromQL"), "got: {msg}"); - } -} diff --git a/crates/frontend-promql/src/histogram.rs b/crates/frontend-promql/src/histogram.rs deleted file mode 100644 index f3f971a55..000000000 --- a/crates/frontend-promql/src/histogram.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Sample-type metadata for the `histogram_quantile` discrimination (issue #79). -//! -//! `histogram_quantile(φ, m)` has two lowerings: exact interpolation over -//! classic cumulative `le` buckets (`AggIntent::HistogramQuantile`, **not** -//! sketch-able) versus the generic sketch-able `Quantile` (native histograms / -//! raw samples, which post-ASAP binding can approximate to an accuracy -//! target). The true -//! signal is the argument's **sample type**, which query structure only -//! *proxies* — see the structural `is_classic_bucket_arg` heuristic, whose -//! false-positive (`…_bucket`-named non-histogram) and false-negative -//! (suffix-less classic histogram) cases this metadata fixes. -//! -//! A client that knows its sample types supplies a [`HistogramCatalog`]; it is -//! consulted first, and the structural heuristic remains the fallback when a -//! metric is undeclared. - -use std::cell::RefCell; -use std::collections::HashMap; - -/// The physical sample type behind a histogram metric — the true signal for -/// whether `histogram_quantile` over it can be re-sketched. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HistogramKind { - /// Classic cumulative `le` buckets — pre-aggregated counts. The - /// distribution can't be reconstructed from them, so it is **not** - /// sketch-able: `histogram_quantile` is exact bucket interpolation. - ClassicBucket, - /// Native (exponential) histogram — sketch-able to an accuracy target. - Native, - /// Raw float samples the client retains — sketch-able. This is the case the - /// generic `Quantile` lowering exists for (a client holding raw samples can - /// build a quantile sketch even though the user wrote `histogram_quantile`). - RawSamples, -} - -impl HistogramKind { - /// Whether `histogram_quantile` over this kind lowers to the sketch-able - /// generic `Quantile` (`true`) rather than exact bucket interpolation. - pub fn is_sketchable(self) -> bool { - !matches!(self, HistogramKind::ClassicBucket) - } -} - -/// Metric-name → declared [`HistogramKind`]. Supplied by a client that knows its -/// sample types, to drive the `histogram_quantile` discrimination from metadata -/// instead of query structure (issue #79). -#[derive(Debug, Clone, Default)] -pub struct HistogramCatalog(HashMap); - -impl HistogramCatalog { - pub fn new() -> Self { - Self::default() - } - - /// Declare `metric`'s sample type (builder style). - pub fn with(mut self, metric: impl Into, kind: HistogramKind) -> Self { - self.0.insert(metric.into(), kind); - self - } - - /// The declared kind for `metric`, if any. - pub fn kind_of(&self, metric: &str) -> Option { - self.0.get(metric).copied() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -/// RAII guard installing `catalog` as the ambient histogram catalog for the -/// current thread, restoring the prior value on drop. -/// -/// Lowering is synchronous and processes one query at a time, so a thread-local -/// ambient catalog cleanly injects this read-only metadata into the deep, -/// free-function `walk` recursion without threading a parameter through every -/// signature (the discrimination is consulted in exactly one place, -/// `walk_histogram`). -pub(crate) struct CatalogGuard(Option); - -impl CatalogGuard { - pub(crate) fn install(catalog: HistogramCatalog) -> Self { - let prev = CURRENT.with(|c| c.borrow_mut().replace(catalog)); - CatalogGuard(prev) - } -} - -impl Drop for CatalogGuard { - fn drop(&mut self) { - CURRENT.with(|c| *c.borrow_mut() = self.0.take()); - } -} - -/// The ambient catalog's declared kind for `metric`, or `None` when no catalog -/// is installed or the metric is undeclared (→ fall back to the heuristic). -pub(crate) fn current_kind_of(metric: &str) -> Option { - CURRENT.with(|c| c.borrow().as_ref().and_then(|cat| cat.kind_of(metric))) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn only_classic_buckets_are_not_sketchable() { - assert!(!HistogramKind::ClassicBucket.is_sketchable()); - assert!(HistogramKind::Native.is_sketchable()); - assert!(HistogramKind::RawSamples.is_sketchable()); - } - - #[test] - fn catalog_lookup() { - let cat = HistogramCatalog::new() - .with("classic", HistogramKind::ClassicBucket) - .with("raw", HistogramKind::RawSamples); - assert_eq!(cat.kind_of("classic"), Some(HistogramKind::ClassicBucket)); - assert_eq!(cat.kind_of("raw"), Some(HistogramKind::RawSamples)); - assert_eq!(cat.kind_of("unknown"), None); - } - - #[test] - fn guard_installs_and_restores_the_ambient_catalog() { - assert_eq!(current_kind_of("m"), None); - { - let _g = CatalogGuard::install( - HistogramCatalog::new().with("m", HistogramKind::ClassicBucket), - ); - assert_eq!(current_kind_of("m"), Some(HistogramKind::ClassicBucket)); - } - // Restored to empty after the guard drops. - assert_eq!(current_kind_of("m"), None); - } -} diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs deleted file mode 100644 index 348fa0263..000000000 --- a/crates/frontend-promql/src/lib.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! PromQL front end: parse (via `promql-parser`) → the canonical, unresolved -//! shape, built directly (issue #179) → [`resolve_root`]. -//! -//! Emits [`UnresolvedQueryExpr`](asap_types::pre_asap::UnresolvedQueryExpr) itself — the -//! canonical `QueryExpr`, generic over an unresolved -//! [`ColumnRef`](asap_types::pre_asap::ColumnRef) — directly, rather than a -//! separate per-language relational DAG; `resolve_root` runs the -//! [`SchemaResolver`](asap_types::pre_asap::SchemaResolver) for positional name resolution. -//! Depends on the PromQL parser only — never on the SQL / DataFusion stack. - -pub mod error; -pub mod histogram; -pub mod promql; - -use asap_types::pre_asap::resolve_root; -use asap_types::pre_asap::QueryExpr; -use asap_types::workload::{DurationMs, PlanningWorkload, QueryLanguage, WorkloadError}; - -pub use error::PromqlError; -pub use histogram::{HistogramCatalog, HistogramKind}; - -/// Lower every normalized PromQL workload entry to a plan-ready `QueryExpr`. -/// -/// PromQL workloads must declare a non-zero `data_ingestion_interval`; it is -/// injected around each bare instant selector. Explicit range selectors keep -/// their query-specified range. -/// `now_ms` is the planning time in Unix milliseconds; cadence evidence must -/// be valid at that time, using the same clock as downstream planning. -pub fn lower_promql_workload( - workload: &PlanningWorkload, - now_ms: u64, -) -> Result, PromqlError> { - lower_promql_workload_inner(workload, now_ms) -} - -/// Like [`lower_promql_workload`], but uses `histograms` to distinguish classic -/// bucket interpolation from generic sketchable quantiles. -pub fn lower_promql_workload_with_histograms( - workload: &PlanningWorkload, - histograms: HistogramCatalog, - now_ms: u64, -) -> Result, PromqlError> { - let _guard = histogram::CatalogGuard::install(histograms); - lower_promql_workload_inner(workload, now_ms) -} - -fn lower_promql_workload_inner( - workload: &PlanningWorkload, - now_ms: u64, -) -> Result, PromqlError> { - if !matches!(workload.query_workload.language, QueryLanguage::PromQL) { - return Err(PromqlError::WrongLanguage(format!( - "{:?}", - workload.query_workload.language - ))); - } - workload.validate()?; - let &DurationMs(interval_ms) = workload - .data_workload - .as_ref() - .expect("validated PromQL workload has data_workload") - .data_ingestion_interval - .value_at(now_ms) - .ok_or(WorkloadError::UnavailableDataIngestionInterval)?; - workload - .query_workload - .entries() - .map(|entry| { - let unresolved = promql::PromqlLowerer::lower_with_ingestion_interval( - &entry.query.0, - &entry.requirements.accuracy.target(), - std::time::Duration::from_millis(interval_ms), - )?; - Ok(resolve_root(&unresolved)?) - }) - .collect() -} - -#[cfg(test)] -mod tests { - // Expiring evidence without an observation timestamp is never usable. - #[test] - fn rejects_unusable_ingestion_evidence() { - let mut input = workload("sum(data)"); - input - .data_workload - .as_mut() - .unwrap() - .data_ingestion_interval - .valid_for_ms = Some(100); - assert!(lower_promql_workload(&input, 0).is_err()); - } - - // Cadence expiry is inclusive; future and expired evidence cannot set a horizon. - #[test] - fn ingestion_evidence_respects_planning_time_with_and_without_histograms() { - let mut input = workload("sum(data)"); - let evidence = &mut input - .data_workload - .as_mut() - .unwrap() - .data_ingestion_interval; - evidence.observed_at_ms = Some(1_000); - evidence.valid_for_ms = Some(100); - for (now_ms, usable) in [(999, false), (1_000, true), (1_100, true), (1_101, false)] { - assert_eq!(lower_promql_workload(&input, now_ms).is_ok(), usable); - assert_eq!( - lower_promql_workload_with_histograms(&input, HistogramCatalog::default(), now_ms) - .is_ok(), - usable - ); - } - input - .data_workload - .as_mut() - .unwrap() - .data_ingestion_interval - .observed_at_ms = None; - assert!( - lower_promql_workload_with_histograms(&input, HistogramCatalog::default(), 1_000) - .is_err() - ); - } - use std::time::Duration; - - use asap_types::pre_asap::QueryExpr; - use asap_types::workload::{ - BatchEntry, DataWorkload, Evidence, PlanningWorkload, Query, QueryRequirements, - QueryWorkload, TimeSelection, - }; - - use super::*; - - fn workload(query: &str) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements::default(), - predictability: Default::default(), - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload { - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - ..Default::default() - }, - ..Default::default() - }), - } - } - - #[test] - fn instant_selector_uses_declared_ingestion_interval() { - let query = lower_promql_workload(&workload("sum by (job) (data)"), 0).unwrap(); - let QueryExpr::Aggregate { child, .. } = &query[0] else { - panic!("expected aggregate") - }; - assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { range, child } - if *range == Duration::from_secs(1) && matches!(child.as_ref(), QueryExpr::Scan { .. })) - ); - } - - #[test] - fn explicit_range_selector_keeps_its_query_range() { - let query = lower_promql_workload(&workload("sum_over_time(data[5m])"), 0).unwrap(); - let QueryExpr::Aggregate { child, .. } = &query[0] else { - panic!("expected aggregate") - }; - assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { range, child } - if *range == Duration::from_secs(300) && matches!(child.as_ref(), QueryExpr::Scan { .. })) - ); - } - - #[test] - fn workload_without_interval_fails_loudly() { - let mut workload = workload("sum(data)"); - workload.data_workload = Some(DataWorkload::default()); - assert!(matches!( - lower_promql_workload(&workload, 0), - Err(PromqlError::InvalidWorkload( - asap_types::workload::WorkloadError::MissingDataIngestionInterval - )) - )); - } -} - -/// Unified lowering, promoted to the root API at planner cutover. -pub mod unified; diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs deleted file mode 100644 index 83251053e..000000000 --- a/crates/frontend-promql/src/promql.rs +++ /dev/null @@ -1,2169 +0,0 @@ -//! PromQL string → the canonical, unresolved -//! [`UnresolvedQueryExpr`](asap_types::pre_asap::query_expr::UnresolvedQueryExpr) -//! (`QueryExpr`). -//! -//! - **Parsing** is delegated to `promql-parser` 0.8. -//! - **Lowering** builds *directly in canonical shape* here (issue #179): the -//! walk interprets PromQL semantics (range vectors, aggregate operators, -//! label matchers) and emits `UnresolvedQueryExpr` nodes with unresolved -//! `ColumnRef`s — the same DAG shape -//! [`resolve_root`](asap_types::pre_asap::resolve_root) later binds to -//! canonical, positional `QueryExpr`. The structural decisions a -//! separate converter stage would otherwise have to make (heavy-hitter -//! `topk` recognition, the `PerEntity`/`Reduce` reduction choice, -//! `without(...)` grouping) are made right here, since a front end -//! building this shape already knows the answer at parse time — see -//! `reduction_for` and `mark_without`. `resolve_root` is left with exactly -//! the schema-*dependent* work: binding every `ColumnRef` to its -//! positional `ColumnId`. -//! -//! # PromQL → canonical unresolved-DAG mapping (summary) -//! -//! | PromQL | Canonical shape | -//! |---|---| -//! | `quantile_over_time(φ, m{f}[w])` | `Aggregate{[Quantile(φ)], TimeRange{w, Scan{predicates}}}` | -//! | `histogram_quantile(φ, )` | `Aggregate{without(le), [HistogramQuantile(φ, le)]}` — cumulative-bucket interpolation (classic form recognised by `by (le)` / a `_bucket` metric / an `le` matcher) | -//! | `histogram_quantile(φ, )` | `Aggregate{[Quantile(φ)]}` over the fully-lowered arg (generic, sketch-able with an accuracy target) | -//! | `histogram_quantiles(v, "l", φ…)` | `Concat{PromqlRelabel{l=φᵢ, }…}` — one branch per φ (issue #109) | -//! | `histogram_count/sum/avg/stddev/stdvar(v)`, `histogram_fraction(l,u,v)` | `Aggregate{[Histogram*]}` — per-series native-histogram accessors (issue #43) | -//! | `OUTER_op(inner_func(m[w]))` (e.g. `sum(rate(m[w]))`) | `Aggregate{[OUTER_op]}` over `Aggregate{[inner_func]}` — two levels | -//! | `OUTER_op()` (e.g. `max(sum by (job) (rate(m[w])))`, `sum(rate(a[w]) + rate(b[w]))`) | `Aggregate{[OUTER_op]}` over the fully-lowered `` — arbitrary function nesting (issue #27) | -//! | `topk(k, )` / `bottomk(k, )` | `Sort{value} → Limit{k}` over the fully-lowered argument | -//! | `avg/min/max/sum_over_time(m[w])` | `Aggregate{[Avg/Min/Max/Sum], TimeRange{w}}` | -//! | `stddev/stdvar_over_time(m[w])` | `Aggregate{[StdDev/Variance], TimeRange{w}}` | -//! | `count_over_time(m[w])` | `Aggregate{[Count], TimeRange{w}}` | -//! | `last/first/mad/ts_of_min/ts_of_max/ts_of_first/ts_of_last_over_time(m[w])` | `Aggregate{[Last/First/Mad/TsOf…OverTime], TimeRange{w}}` — per-series range reducers (issue #51) | -//! | `sort`/`sort_desc(v)`, `sort_by_label[_desc](v,"l"…)` | `Sort{value \| label…}` (no `Limit`) — row-preserving reorder (issue #51); `min_of`/`max_of` scalar reducers → #89 | -//! | `rate(m[w])` / `irate(m[w])` | `Aggregate{[Rate/IRate], TimeRange{w}}` — distinct function identities; shared physical machinery is a later realization choice | -//! | `increase(m[w])` | `Aggregate{[Increase], TimeRange{w}}` | -//! | `changes`/`delta`/`idelta`/`deriv`/`resets`/`predict_linear`/`double_exponential_smoothing`(`m[w]`, …) | `Aggregate{[Changes/Delta/…], TimeRange{w}}` — per-series counter-derivative intents (issue #44) | -//! | `absent(v)` / `absent_over_time(m[w])` / `present_over_time(m[w])` | `Aggregate{[Absent/AbsentOverTime/PresentOverTime]}` — presence intents; the empty→synthesized-sample logic is a post-ASAP concern (issue #47) | -//! | `abs`/`ceil`/`sqrt`/`ln`/`clamp*`/`round`/trig(`v`), `pi()` | `Aggregate{[Math(f)]}` element-wise transform (issue #45); `pi()` → a `PromqlScalarBridge` leaf | -//! | `time()` / `timestamp`/`hour`/`day_of_week`/… (`v`) | `EvalTimestamp` leaf / `Aggregate{[TimeFn(f)]}` (issue #46) | -//! | `vector(s)` / `scalar(v)` | `PromqlVectorFromScalar` / `PromqlScalarFromVector` — the scalar⇄vector bridges (issue #48) | -//! | `label_replace(v,…)` / `label_join(v,…)` | `PromqlRelabel{dst, value}` — per-series label rewrite; value unchanged (issue #50) | -//! | `info(v, [selector])` | `PromqlInfoEnrich{selector}` — label-enrichment join against the info metric(s); join keys resolved during post-ASAP binding (issue #84) | -//! | `group` / `offset` / `@` / `info` | **rejected** — distinct semantics with no intent-algebra representation yet (`info` label-join → #84) | -//! | `OUTER by (dims) (…)` | `Aggregate.reduction = Reduce(by = dims)` (generic `topk by`/`bottomk` grouping → `Sort.partition_by`) | -//! | `count by (d) (…)` | `Aggregate{[Count], …}` | -//! | `group(v)` / `count_values("l", v)` | `Aggregate{[Group]}` (constant 1) / `Aggregate{[CountValues{l}]}` (group-by-value + count, new label `l`) — issue #49 | -//! | `limitk(k, v)` / `limit_ratio(r, v)` | `PromqlSeriesSample{LimitK(k) \| LimitRatio(r)}` — series-sampling selection, whole series kept unchanged (issue #86) | -//! | `topk(k, count_over_time(…))` / `topk(k, sum_over_time(…))` | `Aggregate{[TopK{k}]}` (heavy-hitter intent) over the explicit inner `Aggregate{[Count/Sum]}` | -//! | `topk(k, )` / `bottomk(k, …)` | `Sort{value} → Limit{k}` | -//! | `m{f}` | `Scan{predicates}` | -//! | `a OP b` | `BinaryOp{vector_match}` | -//! | `expr[r:res]` | `PromqlSubquery{r, res}` | -//! | ` offset ` / ` @ `/`start()`/`end()` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | - -use std::rc::Rc; -use std::time::{Duration, SystemTime}; - -use promql_parser::label::{MatchOp, Matcher}; -use promql_parser::parser::{ - self, token, AggregateExpr, AtModifier as ParserAtModifier, BinaryExpr, Call, Expr, - LabelModifier, Offset, VectorMatchCardinality, VectorSelector, -}; - -use asap_types::pre_asap::agg_intent::{topk, AggIntent, MathFunc, TimeFunc}; -use asap_types::pre_asap::query_expr::{ - AtModifier, BinaryOpKind, GroupKeys, GroupSide, Predicate, PromQLVectorSetOpKind, Reduction, - SortKey, Source, TimeShift, UnresolvedQueryExpr as Unresolved, VectorGrouping, VectorMatch, - VectorMatchKind, -}; -use asap_types::pre_asap::{ - ArithmeticOpKind, ColumnRef, CompareOpKind, InfoMatcher, SampleKind, ScalarValue, -}; -use asap_types::types::AccuracyTarget; - -use crate::error::PromqlError as LoweringError; - -type Result = std::result::Result; - -/// Parses and lowers (→ the canonical, unresolved DAG) a PromQL query string. -pub(crate) struct PromqlLowerer; - -#[derive(Debug, Clone)] -enum Outer { - None, - Plain(OuterIntent), - Count, - /// `count_values("l", v)` — group by value + count, emitting the value as a - /// new label `l` (issue #49). - CountValues { - label: String, - }, - TopK { - k: u64, - descending: bool, - }, - /// `limitk`/`limit_ratio` — series-sampling selection (issue #86). - Sample { - kind: SampleKind, - }, -} - -#[derive(Debug, Clone)] -enum OuterIntent { - Sum, - Avg, - Min, - Max, - StdDev, - Variance, - Quantile(f64), - /// `group(v)` — constant 1 per group (issue #49). - Group, -} - -#[derive(Debug, Clone)] -enum InnerFunc { - FrequencyL2, - FrequencyEntropy, - Cardinality, - Quantile(f64), - Avg, - Min, - Max, - Sum, - StdDev, - Variance, - Count, - // `Rate`/`Increase` carry no window of their own — unlike the old Unresolved - // `AggFunc::Rate{window}`, canonical `AggIntent::Rate`/`Increase` have no - // window field either; `windowed_aggregate` reads `Inner.window` - // uniformly for every intent, so it would be a redundant duplicate here. - Rate, - IRate, - Increase, - // Counter-derivative range functions (issue #44). The window rides on the - // enclosing `TimeRange` node (like `*_over_time`), so these carry only - // their non-window scalar params. - Changes, - Delta, - IDelta, - Deriv, - Resets, - PredictLinear(f64), - DoubleExp { smoothing: f64, trend: f64 }, - // Additional range-vector reducers (issue #51). Per-series over the window - // (like `*_over_time`); the window rides on the enclosing Unresolved `Window`. - LastOverTime, - FirstOverTime, - MadOverTime, - TsOfMinOverTime, - TsOfMaxOverTime, - TsOfFirstOverTime, - TsOfLastOverTime, -} - -struct Inner { - metric: String, - matchers: Vec, - window: Option, - func: Option, - /// `offset` / `@` on the selector, carried to the `Source` (issue #40). - shift: TimeShift, -} - -/// Maximum PromQL expression nesting depth the walker accepts. Real queries -/// nest only a handful deep; this bounds the recursive descent (`walk` and the -/// mutually-recursive helpers) so a pathologically nested query is rejected -/// rather than overflowing the stack. -const MAX_DEPTH: usize = 256; - -impl PromqlLowerer { - pub(crate) fn lower_with_ingestion_interval( - query: &str, - accuracy: &AccuracyTarget, - interval: Duration, - ) -> Result { - let _guard = AccuracyGuard::install(accuracy.clone()); - let _interval = IngestionIntervalGuard::install(interval); - let ast = parser::parse(query).map_err(LoweringError::Parse)?; - check_depth(&ast, MAX_DEPTH)?; - walk(&ast) - } -} - -std::thread_local! { - static ACCURACY: std::cell::RefCell = - const { std::cell::RefCell::new(AccuracyTarget::Exact) }; - static INGESTION_INTERVAL: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; -} - -/// RAII guard installing `accuracy` as the ambient accuracy target for the -/// current thread's lowering, restoring the prior value on drop — same shape -/// as `histogram::CatalogGuard`. -struct AccuracyGuard(AccuracyTarget); - -impl AccuracyGuard { - fn install(accuracy: AccuracyTarget) -> Self { - let prev = ACCURACY.with(|a| a.replace(accuracy)); - AccuracyGuard(prev) - } -} - -impl Drop for AccuracyGuard { - fn drop(&mut self) { - ACCURACY.with(|a| *a.borrow_mut() = std::mem::replace(&mut self.0, AccuracyTarget::Exact)); - } -} - -/// The ambient accuracy target installed by the current [`PromqlLowerer::lower`] call. -fn current_accuracy() -> AccuracyTarget { - ACCURACY.with(|a| a.borrow().clone()) -} - -struct IngestionIntervalGuard(Option); - -impl IngestionIntervalGuard { - fn install(interval: Duration) -> Self { - Self(INGESTION_INTERVAL.with(|current| current.replace(Some(interval)))) - } -} - -impl Drop for IngestionIntervalGuard { - fn drop(&mut self) { - INGESTION_INTERVAL.with(|current| *current.borrow_mut() = self.0.take()); - } -} - -fn current_ingestion_interval() -> Duration { - INGESTION_INTERVAL.with(|current| { - current - .borrow() - .expect("ingestion interval is installed for workload lowering") - }) -} - -/// Bounded depth check over the parser AST: errors once nesting would exceed -/// `budget` frames, descending into every child expression. -fn check_depth(expr: &Expr, budget: usize) -> Result<()> { - let Some(budget) = budget.checked_sub(1) else { - return Err(LoweringError::UnsupportedFeature(format!( - "query nesting exceeds the {MAX_DEPTH}-level limit" - ))); - }; - match expr { - Expr::Aggregate(a) => { - check_depth(&a.expr, budget)?; - if let Some(p) = &a.param { - check_depth(p, budget)?; - } - } - Expr::Unary(u) => check_depth(&u.expr, budget)?, - Expr::Binary(b) => { - check_depth(&b.lhs, budget)?; - check_depth(&b.rhs, budget)?; - } - Expr::Paren(p) => check_depth(&p.expr, budget)?, - Expr::Subquery(s) => check_depth(&s.expr, budget)?, - Expr::Call(c) => { - for arg in &c.args.args { - check_depth(arg, budget)?; - } - } - Expr::MatrixSelector(_) - | Expr::VectorSelector(_) - | Expr::NumberLiteral(_) - | Expr::StringLiteral(_) - | Expr::Extension(_) => {} - } - Ok(()) -} - -fn walk(expr: &Expr) -> Result { - match expr { - Expr::Aggregate(agg) => walk_aggregate(agg), - Expr::Call(call) if call.func.name.starts_with("histogram_") => walk_histogram(call), - Expr::Call(call) if is_math_fn(call.func.name) => walk_math(call), - Expr::Call(call) if is_presence_fn(call.func.name) => walk_presence(call), - Expr::Call(call) if is_time_fn(call.func.name) => walk_time(call), - Expr::Call(call) if is_typeconv_fn(call.func.name) => walk_typeconv(call), - Expr::Call(call) if is_label_fn(call.func.name) => walk_label(call), - Expr::Call(call) if is_sort_fn(call.func.name) => walk_sort(call), - // A bare `min_of`/`max_of(consts…)` scalar query folds to a `PromqlScalarBridge` - // leaf; a non-constant argument makes `num_expr` fail → rejected (#89). - Expr::Call(call) if is_scalar_reducer_fn(call.func.name) => { - Ok(Unresolved::promql_scalar(num_expr(expr)?)) - } - Expr::Call(call) if call.func.name == "info" => walk_info(call), - Expr::Call(call) => walk_call(call), - Expr::Binary(bin) => walk_binary(bin), - Expr::Paren(p) => walk(&p.expr), - // `UnaryExpr` is built only by negation (`Neg`); unary `+` is folded to - // identity and `-` to a negated `NumberLiteral`, so this wraps a - // sub-expression whose samples must be sign-flipped. Now that a scalar - // operand exists (#35), express it as `x * -1` — a constant-foldable - // operand (`-(10*1024)`) collapses to a negated `PromqlScalarBridge` leaf; anything - // else is a vector, sign-flipped by a `Mul` against `PromqlScalarBridge(-1)`. `Mul` - // is commutative, so operand order carries no hazard (#36). - Expr::Unary(u) => match num_expr(&u.expr) { - Ok(v) => Ok(Unresolved::promql_scalar(-v)), - Err(_) => Ok(Unresolved::BinaryOp { - op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - lhs: Rc::new(walk(&u.expr)?), - rhs: Rc::new(Unresolved::promql_scalar(-1.0)), - vector_match: None, - }), - }, - Expr::Subquery(sq) => { - let subquery = Unresolved::PromqlSubquery { - range: sq.range, - resolution: sq.step, - child: Rc::new(walk(&sq.expr)?), - }; - // `offset`/`@` move the whole subquery, including its step grid. - let shift = time_shift(sq.offset.as_ref(), sq.at.as_ref())?; - Ok(if shift.is_identity() { - subquery - } else { - Unresolved::TimeShift { - shift, - child: Rc::new(subquery), - } - }) - } - Expr::VectorSelector(vs) => { - let (metric, matchers, shift) = vs_parts(vs)?; - Ok(instant_source(metric, matchers, shift)) - } - Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok(Unresolved::TimeRange { - range: ms.range, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) - } - // A number literal is a scalar leaf (`v > 5`, or a bare scalar query - // `5`). String literals only appear as function args (`label_replace`, - // …), which are not supported, so reject them (issue #35). - Expr::NumberLiteral(n) => Ok(Unresolved::promql_scalar(n.val)), - Expr::StringLiteral(_) => Err(LoweringError::UnsupportedFeature( - "bare string literal".into(), - )), - Expr::Extension(_) => Err(LoweringError::UnsupportedFeature( - "extension expression".into(), - )), - } -} - -/// Lower a bare function call (`rate(m[5m])`, `max_over_time(m[5m])`, …). -/// -/// The common case routes through the flat `lower_inner_call` template. The one -/// exception is a `*_over_time`/`quantile_over_time` function applied to a -/// **sub-query** (`max_over_time(rate(m[5m])[1h:])`): its argument is a -/// `PromQLSubquery`, not a matrix selector, so the flat template's -/// `extract_matrix` can't accept it. Lower the sub-query recursively and reduce -/// it per series (issue #27). -fn walk_call(call: &Call) -> Result { - if let Some(dag) = range_fn_over_subquery(call)? { - return Ok(dag); - } - build(lower_inner_call(call)?, vec![], Outer::None) -} - -/// A range-vector function applied to a **sub-query** — `f([range:res])`. -/// -/// Covers the whole range-vector family: the `*_over_time` reducers, -/// `rate`/`irate`/`increase`, and the counter-derivatives -/// (`changes`/`delta`/`idelta`/`deriv`/`resets`/`predict_linear`/ -/// `double_exponential_smoothing`). Each lowers to a per-series `Aggregate{[f]}` -/// directly over the `PromqlSubquery` — the sub-query is the range context, so -/// there is no separate `Window`/`TimeRange` (this walk treats the `PromqlSubquery` -/// node itself as the range marker). Returns `None` when `call` isn't a range -/// function or its argument isn't a sub-query, so the flat matrix-selector -/// template still handles `f(m[w])` (issues #42, #55). -fn range_fn_over_subquery(call: &Call) -> Result> { - // `rate`/`increase`/`irate` carry their window in the `AggFunc`; over a - // sub-query that window is the sub-query's own range. - if let "rate" | "irate" | "increase" = call.func.name { - let arg_expr = arg(call, 0)?; - if subquery_range(arg_expr).is_none() { - return Ok(None); - } - let inner = match call.func.name { - "rate" => InnerFunc::Rate, - "irate" => InnerFunc::IRate, - "increase" => InnerFunc::Increase, - _ => unreachable!(), - }; - return Ok(Some(per_series_aggregate( - vec![], - inner_intent(&inner), - walk(arg_expr)?, - ))); - } - - // `*_over_time` reducers + counter-derivatives: the func-kind, and the index - // of the matrix/sub-query argument (`quantile_over_time` reads φ from arg 0, - // so its matrix is arg 1; the rest take arg 0 + trailing scalar params). - let (inner, matrix_idx): (InnerFunc, usize) = match call.func.name { - "avg_over_time" => (InnerFunc::Avg, 0), - "min_over_time" => (InnerFunc::Min, 0), - "max_over_time" => (InnerFunc::Max, 0), - "sum_over_time" => (InnerFunc::Sum, 0), - "stddev_over_time" => (InnerFunc::StdDev, 0), - "stdvar_over_time" => (InnerFunc::Variance, 0), - "count_over_time" => (InnerFunc::Count, 0), - "distinct_over_time" => (InnerFunc::Cardinality, 0), - "entropy_over_time" => (InnerFunc::FrequencyEntropy, 0), - "l2_over_time" => (InnerFunc::FrequencyL2, 0), - "quantile_over_time" => (InnerFunc::Quantile(quantile_param(num_arg(call, 0)?)?), 1), - "changes" => (InnerFunc::Changes, 0), - "delta" => (InnerFunc::Delta, 0), - "idelta" => (InnerFunc::IDelta, 0), - "deriv" => (InnerFunc::Deriv, 0), - "resets" => (InnerFunc::Resets, 0), - "last_over_time" => (InnerFunc::LastOverTime, 0), - "first_over_time" => (InnerFunc::FirstOverTime, 0), - "mad_over_time" => (InnerFunc::MadOverTime, 0), - "ts_of_min_over_time" => (InnerFunc::TsOfMinOverTime, 0), - "ts_of_max_over_time" => (InnerFunc::TsOfMaxOverTime, 0), - "ts_of_first_over_time" => (InnerFunc::TsOfFirstOverTime, 0), - "ts_of_last_over_time" => (InnerFunc::TsOfLastOverTime, 0), - "predict_linear" => (InnerFunc::PredictLinear(num_arg(call, 1)?), 0), - "double_exponential_smoothing" => ( - InnerFunc::DoubleExp { - smoothing: num_arg(call, 1)?, - trend: num_arg(call, 2)?, - }, - 0, - ), - _ => return Ok(None), - }; - let arg_expr = arg(call, matrix_idx)?; - if !is_subquery(arg_expr) { - return Ok(None); - } - Ok(Some(per_series_aggregate( - vec![], - inner_intent(&inner), - walk(arg_expr)?, - ))) -} - -/// A (parenthesised) PromQL sub-query — `[range:res]`. -fn is_subquery(expr: &Expr) -> bool { - subquery_range(expr).is_some() -} - -/// The `range` of a (parenthesised) sub-query argument, if it is one. -fn subquery_range(expr: &Expr) -> Option { - match expr { - Expr::Subquery(sq) => Some(sq.range), - Expr::Paren(p) => subquery_range(&p.expr), - _ => None, - } -} - -fn walk_aggregate(agg: &AggregateExpr) -> Result { - let (keys, without) = resolve_group(agg)?; - let outer = outer_kind(agg)?; - - // `without(...)` grouping is modelled only for the reducing aggregations - // (sum/avg/count/…), whose grouping lives on an `Aggregate` node. `topk`/ - // `bottomk` (→ `Sort.partition_by`) and `limitk`/`limit_ratio` (→ `PromqlSeriesSample`) - // would need without-partitioning too; reject rather than silently lower - // them as a `by` grouping (issue #39). - if without && matches!(outer, Outer::TopK { .. } | Outer::Sample { .. }) { - return Err(LoweringError::UnsupportedFeature( - "`without(...)` is only supported on reducing aggregations, not \ - topk/bottomk/limitk" - .into(), - )); - } - - // Fast path — the argument is a bare selector or a single range-vector - // function (`rate`/`increase`/`*_over_time`). `lower_inner` lowers it via the - // flat selector/call template, which also recognises the heavy-hitter - // `topk(k, count_over_time(...))` shape. This is the common two-level case - // (`sum by (job) (rate(m[5m]))`). - // - // General nesting — the argument is itself a composite expression: another - // aggregate (`max(sum by (job) (rate(m[5m])))`), a binary op, a sub-query, or - // a function lowered elsewhere. Lower it recursively with the same `walk` - // used at the top level, then wrap it in the outer aggregation (issue #27; a - // negated argument `sum(-m)` lowers here too, #36). A genuinely unsupported - // inner expression surfaces its own error rather than being mislowered. - // - // Either way, `mark_without` flips the resulting outer `Aggregate` to the - // exclusion form when the modifier was `without(...)`. - let built = match lower_inner(&agg.expr) { - Ok(inner) => build(inner, keys, outer)?, - Err(_) => build_over_sub_dag(outer, keys, walk(&agg.expr)?)?, - }; - Ok(mark_without(built, without)) -} - -/// Map an `AggregateExpr`'s operator (`sum`/`avg`/`topk`/…) to the [`Outer`] -/// shape, independent of what the argument is — so both the flat fast path and -/// the general recursive path share one operator-dispatch. -fn outer_kind(agg: &AggregateExpr) -> Result { - let op = agg.op.id(); - - Ok(if op == token::T_TOPK { - Outer::TopK { - k: count_param(agg)?, - descending: true, - } - } else if op == token::T_BOTTOMK { - Outer::TopK { - k: count_param(agg)?, - descending: false, - } - } else if op == token::T_COUNT { - Outer::Count - } else if op == token::T_SUM { - Outer::Plain(OuterIntent::Sum) - } else if op == token::T_GROUP { - // `group(v)` yields a constant 1 per group (presence), not a sum of - // values — a distinct intent, never folded onto `Sum` (issue #49). - Outer::Plain(OuterIntent::Group) - } else if op == token::T_COUNT_VALUES { - // `count_values("l", v)` groups by sample value and counts, emitting the - // value as a new label `l` (the string parameter) — issue #49. - Outer::CountValues { - label: str_param(agg)?, - } - } else if op == token::T_LIMITK { - // `limitk(k, v)` — up to k series per group (issue #86). - Outer::Sample { - kind: SampleKind::LimitK(count_param(agg)? as usize), - } - } else if op == token::T_LIMIT_RATIO { - // `limit_ratio(r, v)` — an r-fraction of series per group (issue #86). - Outer::Sample { - kind: SampleKind::LimitRatio(ratio_param(agg)?), - } - } else if op == token::T_AVG { - Outer::Plain(OuterIntent::Avg) - } else if op == token::T_MIN { - Outer::Plain(OuterIntent::Min) - } else if op == token::T_MAX { - Outer::Plain(OuterIntent::Max) - } else if op == token::T_STDDEV { - Outer::Plain(OuterIntent::StdDev) - } else if op == token::T_STDVAR { - Outer::Plain(OuterIntent::Variance) - } else if op == token::T_QUANTILE { - Outer::Plain(OuterIntent::Quantile(quantile_param(num_param(agg)?)?)) - } else { - return Err(LoweringError::UnsupportedAggregateOp(format!( - "aggregate token {op}" - ))); - }) -} - -/// Wrap an already-lowered Unresolved sub-DAG in the outer aggregation. This is the -/// general-nesting counterpart to [`build`]: where `build` assembles the -/// two-level shape from a flat [`Inner`], this composes the outer operator over -/// an arbitrary child (`max(sum by (job) (…))`, `sum(a + b)`, …). -/// -/// A heavy-hitter `TopK` is only recognised on the flat `count_over_time` shape -/// (handled in `build`); over a general sub-DAG, `topk`/`bottomk` is a generic -/// order-by-value + limit — the same `Sort{partition_by} → Limit` pair `build` -/// emits for any non-heavy-hitter ranking. -/// Flip the outer `Aggregate` produced for a `without(...)` grouping into the -/// exclusion form. The reducing-aggregation `build` paths place that aggregate -/// at the root; `walk_aggregate` has already rejected the non-aggregate outers -/// (topk/limitk), so a `without` grouping always has an `Aggregate` here (issue -/// #39). A no-op when the modifier was `by`. -/// Flip the outer `Aggregate` produced for a `without(...)` grouping into the -/// exclusion form. A no-op when the modifier was `by`. -/// -/// `reduction_for` (used by [`windowed_aggregate`]/[`outer_aggregate`] to -/// build this node) decides `PerEntity` vs `Reduce(by)` *without* knowing -/// about `without` yet — it only ever sees `by`-mode keys, since `without`'s -/// excluded-labels list is applied here, after the fact, exactly like the -/// pre-#179 legacy `relational::QueryExpr` DAG's own `mark_without` did (its -/// converter read `without` only after this front-end step had already set -/// it). Whether -/// `reduction_for` picked `PerEntity` (only possible when `keys` was empty) -/// or `Reduce(by)`, the correct answer under `without(...)` is always -/// `Reduce(without(keys))`: a `without` grouping is never label-preserving — -/// per-entity requires `!by.is_without()` — so this both re-tags an existing -/// `Reduce` and upgrades a wrongly-early `PerEntity` guess, uniformly. -fn mark_without(dag: Unresolved, without: bool) -> Unresolved { - if !without { - return dag; - } - match dag { - Unresolved::Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => { - let keys = match reduction { - Reduction::Reduce(by) => by.keys().to_vec(), - Reduction::PerEntity => vec![], - }; - Unresolved::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(keys)), - measures, - output_names, - filters, - having, - child, - } - } - other => other, - } -} - -fn build_over_sub_dag(outer: Outer, keys: Vec, child: Unresolved) -> Result { - Ok(match outer { - // `walk_aggregate` always passes a real aggregator; `None` can't occur. - Outer::None => child, - Outer::Plain(intent) => outer_aggregate(keys, outer_intent(&intent), child), - Outer::Count => outer_aggregate(keys, count(), child), - Outer::CountValues { label } => { - outer_aggregate(keys, AggIntent::CountValues { label }, child) - } - Outer::Sample { kind } => Unresolved::PromqlSeriesSample { - by: keys.into(), - kind, - child: Rc::new(child), - }, - Outer::TopK { k, descending } => { - let weighted_counter_ranking = matches!( - &child, - Unresolved::Aggregate { - measures, - child: sum_child, - .. - } if matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && matches!(sum_child.as_ref(), Unresolved::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase])) - ); - let direct_counter_ranking = matches!(&child, Unresolved::Aggregate { - measures, reduction: Reduction::PerEntity, .. - } if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase])); - if descending && (weighted_counter_ranking || direct_counter_ranking) { - return Ok(outer_aggregate( - keys, - AggIntent::TopK { - k: k as usize, - accuracy: current_accuracy(), - }, - child, - )); - } - let sorted = Unresolved::Sort { - keys: vec![SortKey { - expr: Unresolved::Column(ColumnRef::SampleValue), - ascending: !descending, - nulls_first: false, - }], - partition_by: keys.into(), - child: Rc::new(child), - }; - Unresolved::Limit { - n: k as usize, - offset: 0, - child: Rc::new(sorted), - } - } - }) -} - -/// The `histogram_*` function family (issues #43, histogram_quantile). -/// -/// `histogram_quantile(φ, )` lowers `` in full — preserving any -/// `sum by (le)` / `rate` structure inside it. The classic `le`-bucket form -/// becomes [`classic_histogram_quantile`]; a native histogram or raw samples -/// become a `Quantile` over the whole argument. -/// The native-histogram accessors (`histogram_count`/`sum`/`avg`/`stddev`/ -/// `stdvar`/`fraction`) each extract one float per series, lowering to a -/// per-series `Aggregate{[accessor]}` directly over the (instant) argument. -/// `histogram_fraction(lower, upper, v)` reads its bounds from args 0/1 and the -/// vector from arg 2; the rest take the vector at arg 0. -fn walk_histogram(call: &Call) -> Result { - if call.func.name == "histogram_quantiles" { - return walk_histogram_quantiles(call); - } - if call.func.name == "histogram_quantile" { - let phi = quantile_param(num_arg(call, 0)?)?; - let arg_expr = arg(call, 1)?; - // Two lowerings of `histogram_quantile(φ, …)`: - // - classic `le`-bucket form → `HistogramQuantile`, exact interpolation - // over cumulative buckets (not sketch-able). - // - native-histogram / raw-samples form → the generic `Quantile` intent - // (sketch-able). - // The true signal is the argument's sample type: a declared - // `HistogramKind` (issue #79) drives the choice when available, else we - // fall back to the structural `by (le)`/`_bucket` heuristic (issue #43). - if !histogram_arg_is_sketchable(arg_expr) { - return Ok(classic_histogram_quantile(phi, "", walk(arg_expr)?)); - } - let func = AggIntent::Quantile { - col: None, - q: phi, - accuracy: current_accuracy(), - }; - return Ok(outer_aggregate(vec![], func, walk(arg_expr)?)); - } - // (histogram_quantile handled above; accessors below) - let (func, vec_idx) = match call.func.name { - "histogram_count" => (AggIntent::HistogramCount, 0), - "histogram_sum" => (AggIntent::HistogramSum, 0), - "histogram_avg" => (AggIntent::HistogramAvg, 0), - "histogram_stddev" => (AggIntent::HistogramStdDev, 0), - "histogram_stdvar" => (AggIntent::HistogramStdVar, 0), - "histogram_fraction" => ( - AggIntent::HistogramFraction { - lower: num_arg(call, 0)?, - upper: num_arg(call, 1)?, - }, - 2, - ), - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }; - Ok(outer_aggregate(vec![], func, walk(arg(call, vec_idx)?)?)) -} - -/// Classic-bucket `histogram_quantile(φ, child)`. One histogram is the set of -/// series that differ only in `le`, so the aggregate groups `without (le)`. -/// That grouping also seeds `le` into a usage-derived source schema, even -/// when no matcher names it. An empty `output_name` keeps the intent-keyed name. -fn classic_histogram_quantile(q: f64, output_name: &str, child: Unresolved) -> Unresolved { - let le = ColumnRef::Named("le".into()); - Unresolved::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![le.clone()])), - measures: vec![AggIntent::HistogramQuantile { q, le }], - output_names: vec![output_name.into()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -/// `histogram_quantiles(v, "label", φ₀, φ₁, …)` — the experimental multi-quantile -/// form (issue #109). It is `histogram_quantile(φᵢ, v)` fanned out over the -/// quantiles, each branch's output series tagged with `label = φᵢ`. -/// -/// Lowers to a `Concat` of one `PromqlRelabel`-wrapped quantile branch per φ, reusing -/// the single-quantile decision — classic `le`-buckets interpolate -/// (`HistogramQuantile`), native histograms / raw samples take the sketch-able -/// `Quantile` (issues #43 / #79) — so the two functions cannot diverge. -/// -/// The vector argument is lowered once per branch, duplicating the sub-DAG — -/// a future workload-level reuse pass could hoist it back into a single -/// producer. -/// -/// Each branch aliases its value column to `value` rather than taking the -/// intent-keyed name (`quantile_0_5`, `quantile_0_9`, …). `Concat` derives its -/// schema from the first child, so branches that disagree on a column *name* -/// would make the merged schema silently misdescribe every branch but one. The -/// quantile is carried by the `label` column, which is exactly where Prometheus -/// puts it. -fn walk_histogram_quantiles(call: &Call) -> Result { - let vec_expr = arg(call, 0)?; - let label = str_arg(call, 1)?; - if call.args.args.len() < 3 { - return Err(LoweringError::MissingArgument( - "histogram_quantiles(v, label, φ…) needs at least one quantile".into(), - )); - } - // The bucket-vs-native choice is a property of the argument, not of φ. - let sketchable = histogram_arg_is_sketchable(vec_expr); - let branches = (2..call.args.args.len()) - .map(|i| { - let phi = bounded_quantile_param(num_arg(call, i)?)?; - let child = walk(vec_expr)?; - // Each branch aliases its value column to "value" (not the - // intent-keyed default) so `Concat` — which derives its schema - // from the first branch — doesn't silently misdescribe the rest. - let quantile = if sketchable { - let intent = AggIntent::Quantile { - col: None, - q: phi, - accuracy: current_accuracy(), - }; - Unresolved::Aggregate { - reduction: reduction_for(&[], intent.is_per_series()), - measures: vec![intent], - output_names: vec!["value".into()], - filters: vec![], - having: None, - child: Rc::new(child), - } - } else { - classic_histogram_quantile(phi, "value", child) - }; - Ok(Unresolved::PromqlRelabel { - dst: label.clone(), - value: Rc::new(Unresolved::Literal(ScalarValue::Utf8(open_metrics_float( - phi, - )))), - child: Rc::new(quantile), - }) - }) - .collect::>>()?; - // No discriminator asserted here today (issue #228): the φ value each - // branch carries via `PromqlRelabel` *is* structurally a distinct - // per-branch discriminator, but nothing downstream currently needs the - // resulting compound unique key — see - // `docs/design_docs/concat-unique-keys-decision.md`. `Unresolved::concat` - // keeps `output_schema`'s default (drop `unique_keys` entirely). - Ok(Unresolved::concat(branches)) -} - -/// Prometheus's `labels.FormatOpenMetricsFloat` — how `histogram_quantiles` -/// renders each φ into its label value. Go's `%g` shortest round-trip, switching -/// to exponent form outside `[1e-4, 1e21)`, with `.0` appended when the result -/// would otherwise look like an integer. -fn open_metrics_float(v: f64) -> String { - // The cases upstream hardcodes. - if v == 1.0 { - return "1.0".into(); - } - if v == 0.0 { - return "0.0".into(); - } - if v == -1.0 { - return "-1.0".into(); - } - if v.is_nan() { - return "NaN".into(); - } - if v.is_infinite() { - return if v.is_sign_positive() { "+Inf" } else { "-Inf" }.into(); - } - let sci = format!("{v:e}"); - let exp: i32 = sci - .split_once('e') - .and_then(|(_, e)| e.parse().ok()) - .unwrap_or(0); - if !(-4..21).contains(&exp) { - // Go writes a signed, zero-padded two-digit exponent: `1e-05`. - let (mantissa, _) = sci.split_once('e').unwrap_or((sci.as_str(), "0")); - let sign = if exp < 0 { '-' } else { '+' }; - return format!("{mantissa}e{sign}{:02}", exp.abs()); - } - let s = format!("{v}"); - if s.contains(['e', '.']) { - s - } else { - format!("{s}.0") - } -} - -/// The time / calendar functions (issue #46). -fn is_time_fn(name: &str) -> bool { - matches!( - name, - "time" - | "timestamp" - | "minute" - | "hour" - | "day_of_week" - | "day_of_month" - | "day_of_year" - | "month" - | "year" - | "days_in_month" - ) -} - -/// `time()` → the `EvalTimestamp` leaf. `timestamp(v)` and the calendar accessors → -/// `Aggregate{[TimeFn(f)]}` over the argument vector, or over `EvalTimestamp` for the -/// no-argument calendar forms (`hour()`, `day_of_week()`, …). Issue #46. -fn walk_time(call: &Call) -> Result { - if call.func.name == "time" { - return Ok(Unresolved::EvalTimestamp); - } - let func = match call.func.name { - "timestamp" => TimeFunc::Timestamp, - "minute" => TimeFunc::Minute, - "hour" => TimeFunc::Hour, - "day_of_week" => TimeFunc::DayOfWeek, - "day_of_month" => TimeFunc::DayOfMonth, - "day_of_year" => TimeFunc::DayOfYear, - "month" => TimeFunc::Month, - "year" => TimeFunc::Year, - "days_in_month" => TimeFunc::DaysInMonth, - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }; - // A calendar function with no argument reads the evaluation time; otherwise - // it maps over each sample's timestamp in the argument vector. - let inner = if call.args.args.is_empty() { - Unresolved::EvalTimestamp - } else { - walk(arg(call, 0)?)? - }; - Ok(outer_aggregate(vec![], AggIntent::TimeFn(func), inner)) -} - -/// The presence functions (issue #47). -fn is_presence_fn(name: &str) -> bool { - matches!(name, "absent" | "absent_over_time" | "present_over_time") -} - -/// `absent(v)` / `absent_over_time(m[w])` / `present_over_time(m[w])` — lowered -/// to an `Aggregate{[Absent/…]}` over the (instant or range) argument. The -/// empty-result → synthesized-1-sample logic is a post-ASAP/runtime concern; -/// the canonical DAG only marks the operation (issue #47). -fn walk_presence(call: &Call) -> Result { - let func = match call.func.name { - "absent" => AggIntent::Absent, - "absent_over_time" => AggIntent::AbsentOverTime, - "present_over_time" => AggIntent::PresentOverTime, - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }; - // arg 0 is the instant vector (`absent`) or range vector (`*_over_time`); - // `walk` produces a `Window` for the matrix-selector forms. - Ok(outer_aggregate(vec![], func, walk(arg(call, 0)?)?)) -} - -/// The scalar⇄vector type-conversion functions (issue #48). `info` is *not* -/// here: it is a label-enrichment join against info metrics, not a type -/// conversion, so it falls through to the `UnsupportedFunction` path (#84). -fn is_typeconv_fn(name: &str) -> bool { - matches!(name, "vector" | "scalar") -} - -/// `vector(s)` — promote a scalar to a label-less instant vector. `scalar(v)` -/// — collapse a single-element vector to its value. Both are honest bridge -/// nodes in the IR; the "exactly one element → NaN otherwise" runtime rule of -/// `scalar` is a post-ASAP/runtime concern (issue #48). -fn walk_typeconv(call: &Call) -> Result { - let inner = walk(arg(call, 0)?)?; - Ok(match call.func.name { - "vector" => Unresolved::PromqlVectorFromScalar(Rc::new(inner)), - "scalar" => Unresolved::PromqlScalarFromVector(Rc::new(inner)), - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }) -} - -/// The instant-vector reordering functions (issue #51). -fn is_sort_fn(name: &str) -> bool { - matches!( - name, - "sort" | "sort_desc" | "sort_by_label" | "sort_by_label_desc" - ) -} - -/// `sort`/`sort_desc(v)` reorder an instant vector by sample value; -/// `sort_by_label`/`sort_by_label_desc(v, "l"…)` reorder by label values. All -/// lower to a bare `Sort` (no `Limit`) over the vector argument — a faithful, -/// row-preserving reordering (issue #51). -fn walk_sort(call: &Call) -> Result { - let child = Rc::new(walk(arg(call, 0)?)?); - let (by_value, ascending) = match call.func.name { - "sort" => (true, true), - "sort_desc" => (true, false), - "sort_by_label" => (false, true), - "sort_by_label_desc" => (false, false), - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }; - let sort_key = |expr| SortKey { - expr, - ascending, - nulls_first: false, - }; - let keys = if by_value { - vec![sort_key(Unresolved::Column(ColumnRef::SampleValue))] - } else { - // `sort_by_label(v, "l1", "l2", …)` — one key per label arg, in order. - if call.args.args.len() < 2 { - return Err(LoweringError::MissingArgument( - "sort_by_label needs at least one label".into(), - )); - } - (1..call.args.args.len()) - .map(|i| { - Ok(sort_key(Unresolved::Column(ColumnRef::Named(str_arg( - call, i, - )?)))) - }) - .collect::>>()? - }; - Ok(Unresolved::Sort { - keys, - partition_by: GroupKeys::none(), - child, - }) -} - -/// `info(v, [selector])` — a label-enrichment join. Lowers the input vector and -/// wraps it in an `PromqlInfoEnrich` carrying the (optional) data-label selector's -/// matchers; the actual join against the info metric — on shared identifying -/// labels — is resolved during post-ASAP binding (issue #84). -fn walk_info(call: &Call) -> Result { - let child = Rc::new(walk(arg(call, 0)?)?); - let selector = match call.args.args.get(1) { - Some(sel) => info_selector(sel)?, - None => Vec::new(), // default: enrich from `target_info` - }; - Ok(Unresolved::PromqlInfoEnrich { selector, child }) -} - -/// Extract the `info` data-label selector's matchers. Unlike an ordinary -/// selector these are **info-metric-side** and may carry regex / multiple -/// `__name__` matchers (which pick the info metric(s)), so they bypass the -/// single-metric `vs_parts` restriction and are kept symbolic. -fn info_selector(expr: &Expr) -> Result> { - match expr { - Expr::VectorSelector(vs) => Ok(vs - .matchers - .matchers - .iter() - .map(|m| InfoMatcher { - label: m.name.clone(), - op: match &m.op { - MatchOp::Equal => CompareOpKind::Eq, - MatchOp::NotEqual => CompareOpKind::Ne, - MatchOp::Re(_) => CompareOpKind::Regex, - MatchOp::NotRe(_) => CompareOpKind::NotRegex, - }, - value: m.value.clone(), - }) - .collect()), - Expr::Paren(p) => info_selector(&p.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "`info` data-label selector must be a label-matcher set, got `{other}`" - ))), - } -} - -/// The label-rewrite functions (issue #50). -fn is_label_fn(name: &str) -> bool { - matches!(name, "label_replace" | "label_join") -} - -/// `label_replace(v, dst, replacement, src, regex)` / -/// `label_join(v, dst, sep, src…)` — per-series label rewrites. Both lower to a -/// `PromqlRelabel` over the fully-lowered vector argument, differing only in the -/// expression that computes the destination label: `label_replace` a regex -/// capture-expansion, `label_join` a separator-joined concatenation. Sample -/// values are untouched; the regex-match-or-passthrough and capture-expansion -/// are post-ASAP/runtime concerns (issue #50). -fn walk_label(call: &Call) -> Result { - let child = Rc::new(walk(arg(call, 0)?)?); - match call.func.name { - "label_replace" => { - let dst = str_arg(call, 1)?; - let replacement = str_arg(call, 2)?; - let src = str_arg(call, 3)?; - let regex = str_arg(call, 4)?; - let value = Unresolved::FunctionCall { - name: "label_replace".into(), - args: vec![ - Unresolved::Column(ColumnRef::Named(src)), - Unresolved::Literal(ScalarValue::Utf8(regex)), - Unresolved::Literal(ScalarValue::Utf8(replacement)), - ], - }; - Ok(Unresolved::PromqlRelabel { - dst, - value: Rc::new(value), - child, - }) - } - "label_join" => { - // label_join(v, dst, sep, src_1, …, src_n) — needs ≥1 source label. - if call.args.args.len() < 4 { - return Err(LoweringError::MissingArgument( - "label_join(v, dst, sep, src…) needs at least one source label".into(), - )); - } - let dst = str_arg(call, 1)?; - let sep = str_arg(call, 2)?; - let mut args = vec![Unresolved::Literal(ScalarValue::Utf8(sep))]; - for i in 3..call.args.args.len() { - args.push(Unresolved::Column(ColumnRef::Named(str_arg(call, i)?))); - } - let value = Unresolved::FunctionCall { - name: "label_join".into(), - args, - }; - Ok(Unresolved::PromqlRelabel { - dst, - value: Rc::new(value), - child, - }) - } - other => Err(LoweringError::UnsupportedFunction(other.to_string())), - } -} - -/// The element-wise math / trig functions (issue #45). -fn is_math_fn(name: &str) -> bool { - matches!( - name, - "abs" - | "ceil" - | "floor" - | "exp" - | "ln" - | "log2" - | "log10" - | "sqrt" - | "sgn" - | "sin" - | "cos" - | "tan" - | "asin" - | "acos" - | "atan" - | "sinh" - | "cosh" - | "tanh" - | "asinh" - | "acosh" - | "atanh" - | "deg" - | "rad" - | "pi" - | "round" - | "clamp" - | "clamp_min" - | "clamp_max" - ) -} - -/// A math / trig function — a per-series element-wise value transform, lowered -/// to a per-series `Aggregate{[Math(f)]}` over the (instant) argument vector. -/// `pi()` is the constant π, lowered to a `PromqlScalarBridge` leaf (issue #45). -fn walk_math(call: &Call) -> Result { - if call.func.name == "pi" { - return Ok(Unresolved::promql_scalar(std::f64::consts::PI)); - } - let func = match call.func.name { - "abs" => MathFunc::Abs, - "ceil" => MathFunc::Ceil, - "floor" => MathFunc::Floor, - "exp" => MathFunc::Exp, - "ln" => MathFunc::Ln, - "log2" => MathFunc::Log2, - "log10" => MathFunc::Log10, - "sqrt" => MathFunc::Sqrt, - "sgn" => MathFunc::Sgn, - "sin" => MathFunc::Sin, - "cos" => MathFunc::Cos, - "tan" => MathFunc::Tan, - "asin" => MathFunc::Asin, - "acos" => MathFunc::Acos, - "atan" => MathFunc::Atan, - "sinh" => MathFunc::Sinh, - "cosh" => MathFunc::Cosh, - "tanh" => MathFunc::Tanh, - "asinh" => MathFunc::Asinh, - "acosh" => MathFunc::Acosh, - "atanh" => MathFunc::Atanh, - "deg" => MathFunc::Deg, - "rad" => MathFunc::Rad, - // `round(v)` defaults the step to 1; `round(v, to)` reads arg 1. - "round" => MathFunc::Round { - to_nearest: if call.args.args.len() >= 2 { - num_arg(call, 1)? - } else { - 1.0 - }, - }, - "clamp" => MathFunc::Clamp { - min: num_arg(call, 1)?, - max: num_arg(call, 2)?, - }, - "clamp_min" => MathFunc::ClampMin { - min: num_arg(call, 1)?, - }, - "clamp_max" => MathFunc::ClampMax { - max: num_arg(call, 1)?, - }, - other => return Err(LoweringError::UnsupportedFunction(other.to_string())), - }; - // The value being transformed is always arg 0 (a vector). - let inner = walk(arg(call, 0)?)?; - Ok(outer_aggregate(vec![], AggIntent::Math(func), inner)) -} - -/// Whether `expr` is a **classic cumulative-bucket** `histogram_quantile` -/// argument — as opposed to a native histogram or raw samples. Recognised -/// structurally, by any of: -/// - a `by (le)` grouping (`sum by (le) (…)`), -/// - a selector on a classic `_bucket` metric (`http_request_…_bucket`), -/// - a selector with an `le` label matcher (`{le="…"}`). -/// -/// The bucket form must be *interpolated* (`HistogramQuantile`); everything -/// else is a sketch-able generic `Quantile`. This is a heuristic proxy for the -/// real signal — the argument's sample type — which isn't visible at lowering; -/// see the follow-up issue on the discrimination criteria (issue #43). -/// Whether `histogram_quantile(φ, arg)` lowers to the sketch-able generic -/// `Quantile` (`true`) or exact classic-bucket interpolation (`false`). -/// -/// Metadata wins: if any metric referenced in `arg` has a declared -/// [`HistogramKind`](crate::histogram::HistogramKind), that decides it (issue -/// #79) — this fixes both the false-positive (a `…_bucket`-named non-histogram -/// declared `RawSamples`) and the false-negative (a suffix-less classic -/// histogram declared `ClassicBucket`) of the structural heuristic. With no -/// declaration, fall back to the structural `by (le)`/`_bucket` heuristic. -fn histogram_arg_is_sketchable(arg: &Expr) -> bool { - let mut metrics = Vec::new(); - collect_metric_names(arg, &mut metrics); - for metric in &metrics { - if let Some(kind) = crate::histogram::current_kind_of(metric) { - return kind.is_sketchable(); - } - } - !is_classic_bucket_arg(arg) -} - -/// Collect the metric names of every vector/matrix selector reachable in `expr` -/// (for the metadata lookup in [`histogram_arg_is_sketchable`]). Skips -/// name-less selectors like `{le="…"}`. -fn collect_metric_names(expr: &Expr, out: &mut Vec) { - match expr { - Expr::VectorSelector(vs) => { - if let Ok((metric, ..)) = vs_parts(vs) { - if !metric.is_empty() { - out.push(metric); - } - } - } - Expr::MatrixSelector(ms) => { - if let Ok((metric, ..)) = vs_parts(&ms.vs) { - if !metric.is_empty() { - out.push(metric); - } - } - } - Expr::Paren(p) => collect_metric_names(&p.expr, out), - Expr::Unary(u) => collect_metric_names(&u.expr, out), - Expr::Subquery(s) => collect_metric_names(&s.expr, out), - Expr::Aggregate(a) => collect_metric_names(&a.expr, out), - Expr::Binary(b) => { - collect_metric_names(&b.lhs, out); - collect_metric_names(&b.rhs, out); - } - Expr::Call(c) => c - .args - .args - .iter() - .for_each(|a| collect_metric_names(a, out)), - _ => {} - } -} - -fn is_classic_bucket_arg(expr: &Expr) -> bool { - match expr { - Expr::Paren(p) => is_classic_bucket_arg(&p.expr), - Expr::Unary(u) => is_classic_bucket_arg(&u.expr), - Expr::Subquery(s) => is_classic_bucket_arg(&s.expr), - Expr::Aggregate(agg) => { - matches!( - &agg.modifier, - Some(LabelModifier::Include(ls)) if ls.labels.iter().any(|l| l == "le") - ) || is_classic_bucket_arg(&agg.expr) - } - Expr::Binary(b) => is_classic_bucket_arg(&b.lhs) || is_classic_bucket_arg(&b.rhs), - Expr::Call(c) => c.args.args.iter().any(|a| is_classic_bucket_arg(a)), - Expr::VectorSelector(vs) => selector_is_bucket(vs), - Expr::MatrixSelector(ms) => selector_is_bucket(&ms.vs), - _ => false, - } -} - -/// A classic histogram bucket selector — a `_bucket`-named metric (via bare name -/// or `__name__` matcher) or an explicit `le` label matcher. -fn selector_is_bucket(vs: &VectorSelector) -> bool { - let name = vs.name.as_deref().or_else(|| { - vs.matchers - .matchers - .iter() - .find(|m| m.name == "__name__") - .map(|m| m.value.as_str()) - }); - name.is_some_and(|n| n.ends_with("_bucket")) - || vs.matchers.matchers.iter().any(|m| m.name == "le") -} - -fn walk_binary(bin: &BinaryExpr) -> Result { - let lhs = scalar_or_vector(&bin.lhs)?; - let rhs = scalar_or_vector(&bin.rhs)?; - // `VectorMatch` has no fill field; dropping fill would change which series - // are emitted and their values, so the query must fall back to exact - // execution instead. - if let Some(m) = &bin.modifier { - if m.fill_values.lhs.is_some() || m.fill_values.rhs.is_some() { - return Err(LoweringError::UnsupportedFeature(format!( - "`fill` vector-matching modifier: `{bin}`" - ))); - } - } - let op = match (binop(bin.op.id())?, bin.return_bool()) { - (BinaryOpKind::Compare(op), true) => BinaryOpKind::CompareBool(op), - (op, _) => op, - }; - let vector_match = bin.modifier.as_ref().map(|m| { - let (kind, labels) = match &m.matching { - Some(LabelModifier::Include(ls)) => (VectorMatchKind::On, ls.labels.clone()), - Some(LabelModifier::Exclude(ls)) => (VectorMatchKind::Ignoring, ls.labels.clone()), - // No explicit `on(…)`/`ignoring(…)` — the parser attaches a default - // modifier to every set op (`and`/`or`/`unless`). The default is - // "match on all shared labels", which is exactly `ignoring([])` - // (ignore no labels). Representing it as `Ignoring([])` — not - // `On([])` — keeps it distinct from an explicit `on()` (match on the - // empty label set) while making it correctly equal to an explicit - // `ignoring()` (issue #68). - None => (VectorMatchKind::Ignoring, vec![]), - }; - let grouping = match &m.card { - VectorMatchCardinality::ManyToOne(ls) => Some(VectorGrouping { - side: GroupSide::Left, - labels: ls.labels.clone(), - }), - VectorMatchCardinality::OneToMany(ls) => Some(VectorGrouping { - side: GroupSide::Right, - labels: ls.labels.clone(), - }), - _ => None, - }; - VectorMatch { - kind, - labels, - grouping, - } - }); - Ok(Unresolved::BinaryOp { - op, - lhs: Rc::new(lhs), - rhs: Rc::new(rhs), - vector_match, - }) -} - -fn lower_inner(expr: &Expr) -> Result { - match expr { - Expr::VectorSelector(vs) => { - let (metric, matchers, shift) = vs_parts(vs)?; - Ok(Inner { - metric, - matchers, - window: None, - func: None, - shift, - }) - } - Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok(Inner { - metric, - matchers, - window: Some(ms.range), - func: None, - shift, - }) - } - Expr::Paren(p) => lower_inner(&p.expr), - Expr::Call(call) => lower_inner_call(call), - other => Err(LoweringError::UnsupportedFeature(format!( - "aggregate argument: `{other}`" - ))), - } -} - -fn lower_inner_call(call: &Call) -> Result { - let name = call.func.name; - let at0 = |func: InnerFunc| -> Result { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(func), - shift, - }) - }; - match name { - "rate" | "irate" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(if name == "irate" { - InnerFunc::IRate - } else { - InnerFunc::Rate - }), - shift, - }) - } - "increase" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::Increase), - shift, - }) - } - "quantile_over_time" => { - let phi = quantile_param(num_arg(call, 0)?)?; - let (metric, matchers, window, shift) = extract_matrix(arg(call, 1)?)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::Quantile(phi)), - shift, - }) - } - "avg_over_time" => at0(InnerFunc::Avg), - "min_over_time" => at0(InnerFunc::Min), - "max_over_time" => at0(InnerFunc::Max), - "sum_over_time" => at0(InnerFunc::Sum), - "stddev_over_time" => at0(InnerFunc::StdDev), - "stdvar_over_time" => at0(InnerFunc::Variance), - "count_over_time" => at0(InnerFunc::Count), - "distinct_over_time" => at0(InnerFunc::Cardinality), - "entropy_over_time" => at0(InnerFunc::FrequencyEntropy), - "l2_over_time" => at0(InnerFunc::FrequencyL2), - // Counter-derivative range functions (issue #44). Each has its own - // intent — `changes` (value-change count) and `resets` (counter-reset - // count) are NOT sample counts, so they are not aliased to - // `count_over_time`. The window is arg 0's matrix; scalar params follow. - "changes" => at0(InnerFunc::Changes), - "delta" => at0(InnerFunc::Delta), - "idelta" => at0(InnerFunc::IDelta), - "deriv" => at0(InnerFunc::Deriv), - "resets" => at0(InnerFunc::Resets), - // Additional range-vector reducers (issue #51) — same windowed - // per-series shape as the `*_over_time` family above. - "last_over_time" => at0(InnerFunc::LastOverTime), - "first_over_time" => at0(InnerFunc::FirstOverTime), - "mad_over_time" => at0(InnerFunc::MadOverTime), - "ts_of_min_over_time" => at0(InnerFunc::TsOfMinOverTime), - "ts_of_max_over_time" => at0(InnerFunc::TsOfMaxOverTime), - "ts_of_first_over_time" => at0(InnerFunc::TsOfFirstOverTime), - "ts_of_last_over_time" => at0(InnerFunc::TsOfLastOverTime), - "predict_linear" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - let seconds = num_arg(call, 1)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::PredictLinear(seconds)), - shift, - }) - } - "double_exponential_smoothing" => { - let (metric, matchers, window, shift) = extract_matrix(arg(call, 0)?)?; - let smoothing = num_arg(call, 1)?; - let trend = num_arg(call, 2)?; - Ok(Inner { - metric, - matchers, - window: Some(window), - func: Some(InnerFunc::DoubleExp { smoothing, trend }), - shift, - }) - } - other => Err(LoweringError::UnsupportedFunction(other.to_string())), - } -} - -/// Assemble the Layer-2 DAG from a lowered inner vector, the resolved group -/// keys, and the enclosing aggregator shape. -fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { - match outer { - Outer::None => match &inner.func { - None => Ok(instant_source(inner.metric, inner.matchers, inner.shift)), - Some(f) => { - let intent = inner_intent(f); - Ok(windowed_aggregate(inner, keys, intent)) - } - }, - // An OUTER aggregation operator (`sum`/`avg`/…/`count`) over an inner - // range-vector function (`rate`/`increase`/`*_over_time`) is a - // two-level reduction: the inner func runs per series, the outer op - // then aggregates across series. Collapsing them into one aggregate - // silently drops a level — e.g. `sum(rate(m[w]))` must keep the `sum`. - Outer::Plain(intent) => Ok(match &inner.func { - None => windowed_aggregate(inner, keys, outer_intent(&intent)), - Some(f) => { - let inner_i = inner_intent(f); - let inner_agg = windowed_aggregate(inner, vec![], inner_i); - outer_aggregate(keys, outer_intent(&intent), inner_agg) - } - }), - Outer::Count => Ok(match &inner.func { - None => windowed_aggregate(inner, keys, count()), - Some(f) => { - let inner_i = inner_intent(f); - let inner_agg = windowed_aggregate(inner, vec![], inner_i); - outer_aggregate(keys, count(), inner_agg) - } - }), - Outer::CountValues { label } => Ok(match &inner.func { - None => windowed_aggregate(inner, keys, AggIntent::CountValues { label }), - Some(f) => { - let inner_i = inner_intent(f); - let inner_agg = windowed_aggregate(inner, vec![], inner_i); - outer_aggregate(keys, AggIntent::CountValues { label }, inner_agg) - } - }), - Outer::Sample { kind } => { - // Series sampling selects whole series unchanged — like generic - // `topk`, a range-vector argument reduces per series first (label- - // preserving), a bare selector is sampled directly; neither is - // wrapped in a reducing aggregate (issue #86). - let base = match inner.func.as_ref().map(inner_intent) { - Some(intent) => windowed_aggregate(inner, vec![], intent), - None => instant_source(inner.metric, inner.matchers, inner.shift), - }; - Ok(Unresolved::PromqlSeriesSample { - by: keys.into(), - kind, - child: Rc::new(base), - }) - } - Outer::TopK { k, descending } => { - // Preserve the counter-value ranking intent. Physical candidates - // may rebuild a heap over finalized rates or use exact Sort/Limit; - // neither is allowed to sum raw counter samples as ranking weights. - if descending && matches!(inner.func, Some(InnerFunc::Rate | InnerFunc::Increase)) { - let intent = inner_intent(inner.func.as_ref().expect("counter function")); - let ranked = windowed_aggregate(inner, vec![], intent); - return Ok(Unresolved::Aggregate { - reduction: Reduction::Reduce(keys.into()), - measures: vec![AggIntent::TopK { - k: k as usize, - accuracy: current_accuracy(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(ranked), - }); - } - // Heavy-hitter only when ranking by an additive measure (`count` - // or `sum`): that is a - // first-class aggregate intent → `TopK`. Any other ranking (topk - // over avg/quantile, a bare selector's raw value, all bottomk) - // is a generic order-by-value + limit and stays as the `Sort + Limit` - // operator pair. The descending-plus-measure rule is shared with the - // canonicalize-pass promotion so the two cannot drift (issue #38). - let measure = match inner.func { - Some(InnerFunc::Count) => topk::Ranking::Frequency, - Some(InnerFunc::Sum) => topk::Ranking::WeightedSum, - _ => topk::Ranking::NonAdditive, - }; - let additive_ranking = measure.is_supported(descending); - if additive_ranking { - // Preserve the ranked aggregate intent in the canonical DAG so the - // intent algebra is explicit about what is being computed. - // Post-ASAP binding may fuse the Count and TopK into a - // single-pass heavy-hitter sketch (SpaceSaving / - // CMS-with-heap), but that is a cost-model decision, not a - // canonical-IR concern. - let ranked = match measure { - topk::Ranking::Frequency => InnerFunc::Count, - topk::Ranking::WeightedSum => InnerFunc::Sum, - topk::Ranking::NonAdditive => { - unreachable!("heavy-hitter gate rejected non-additive ranking") - } - }; - let ranked_agg = windowed_aggregate(inner, vec![], inner_intent(&ranked)); - Ok(Unresolved::Aggregate { - // A ranking always reduces (a `by`-empty TopK ranks the - // whole input into one ordering, never per-entity). - reduction: Reduction::Reduce(keys.into()), - measures: vec![AggIntent::TopK { - k: k as usize, - accuracy: current_accuracy(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(ranked_agg), - }) - } else { - // The base over which we rank. A range-vector-function argument - // (`topk(k, rate(m[5m]))`) reduces *per series* first — that is - // label-preserving, so the `by (host)` partition labels survive. - // A **bare instant selector** (`topk(k, m)`) ranks its own - // samples directly: it must NOT be wrapped in a reducing - // aggregate. Defaulting it to `Sum` was both semantically wrong - // (PromQL `topk` ranks the raw samples, it does not sum them) and - // destructive — the cross-series `Sum` collapses every label, - // including the `by (…)` partition keys, so they no longer - // resolve (issue #30). Keep the selector label-preserving so - // `Sort.partition_by` can rank within each group (issue #12). - let base = match inner.func.as_ref().map(inner_intent) { - Some(intent) => windowed_aggregate(inner, vec![], intent), - None => instant_source(inner.metric, inner.matchers, inner.shift), - }; - let sorted = Unresolved::Sort { - keys: vec![SortKey { - expr: Unresolved::Column(ColumnRef::SampleValue), - ascending: !descending, - nulls_first: false, - }], - partition_by: keys.into(), - child: Rc::new(base), - }; - Ok(Unresolved::Limit { - n: k as usize, - offset: 0, - child: Rc::new(sorted), - }) - } - } - } -} - -/// Decide `PerEntity` vs `Reduce(by)` for a canonical `Aggregate`, entirely -/// from local PromQL semantics: the keys and whether this operation preserves -/// each input series. It never infers entity reduction from the child DAG's -/// temporal shape. `without()` is applied -/// separately, post-hoc, by `mark_without` — see its doc for why that's still -/// correct here. -fn reduction_for(keys: &[ColumnRef], per_entity: bool) -> Reduction { - if keys.is_empty() && per_entity { - Reduction::PerEntity - } else { - Reduction::Reduce(GroupKeys::by(keys.to_vec())) - } -} - -/// `Aggregate{reduction, [intent]}` over `[TimeRange{w}] → Scan`. Always wraps -/// in `TimeRange` when there's a window — including for `Rate`/`Increase`, -/// whose window rides on `inner.window` too (set redundantly alongside the -/// intent itself): canonical `AggIntent::Rate`/`Increase` carry no window -/// field of their own, unlike the old Unresolved `AggFunc::Rate{window}` — "the range -/// is on the enclosing `TimeRange` node" is now true unconditionally, so -/// there's no more `skip_window` special case. -fn windowed_aggregate( - inner: Inner, - keys: Vec, - intent: AggIntent, -) -> Unresolved { - let base = filtered_source(inner.metric, inner.matchers, inner.shift); - let child = match inner.window { - Some(w) => Unresolved::TimeRange { - range: w, - child: Rc::new(base), - }, - None => base, - }; - let reduction = reduction_for(&keys, inner.window.is_some() || intent.is_per_series()); - let child = if inner.window.is_none() { - Unresolved::TimeRange { - range: current_ingestion_interval(), - child: Rc::new(child), - } - } else { - child - }; - Unresolved::Aggregate { - reduction, - measures: vec![intent], - // A single empty entry — never an override — so the resolver keeps - // PromQL's intent-keyed output names ("sum", "quantile_0_99", …) - // instead. - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -/// `Aggregate{reduction, [intent]}` directly over an existing Unresolved sub-DAG — the -/// OUTER level of a two-level aggregation such as `sum(rate(…))` or the -/// `Aggregate{[Quantile]}` that wraps a `histogram_quantile` argument. -fn outer_aggregate( - keys: Vec, - intent: AggIntent, - child: Unresolved, -) -> Unresolved { - let reduction = reduction_for(&keys, intent.is_per_series()); - Unresolved::Aggregate { - reduction, - measures: vec![intent], - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -/// A temporal range function over a subquery consumes each series' subquery -/// samples independently. Unlike an ordinary outer aggregate, this cannot be -/// inferred from the intent: `max` is cross-series in `max(v)`, but per-series -/// in `max_over_time(v[...])`. -fn per_series_aggregate( - keys: Vec, - intent: AggIntent, - child: Unresolved, -) -> Unresolved { - let reduction = reduction_for(&keys, true); - Unresolved::Aggregate { - reduction, - measures: vec![intent], - output_names: vec![String::new()], - filters: vec![], - having: None, - child: Rc::new(child), - } -} - -fn filtered_source(metric: String, matchers: Vec, shift: TimeShift) -> Unresolved { - let scan = Unresolved::Scan { - source: Source::TimeSeries { metric }, - predicates: matchers - .into_iter() - .map(|m| Predicate(Rc::new(m))) - .collect(), - // Usage-derived (PromQL is schemaless) — the SchemaResolver fills this in. - schema: None, - }; - if shift.is_identity() { - scan - } else { - Unresolved::TimeShift { - shift, - child: Rc::new(scan), - } - } -} - -fn instant_source(metric: String, matchers: Vec, shift: TimeShift) -> Unresolved { - Unresolved::TimeRange { - range: current_ingestion_interval(), - child: Rc::new(filtered_source(metric, matchers, shift)), - } -} - -/// Count vector elements regardless of their sample values. -fn count() -> AggIntent { - AggIntent::Count { - accuracy: current_accuracy(), - } -} - -fn inner_intent(f: &InnerFunc) -> AggIntent { - match f { - InnerFunc::FrequencyL2 => AggIntent::FrequencyL2 { - col: None, - accuracy: current_accuracy(), - }, - InnerFunc::FrequencyEntropy => AggIntent::FrequencyEntropy { - col: None, - accuracy: current_accuracy(), - }, - InnerFunc::Cardinality => AggIntent::Cardinality { - cols: vec![], - accuracy: current_accuracy(), - }, - InnerFunc::Quantile(q) => AggIntent::Quantile { - col: None, - q: *q, - accuracy: current_accuracy(), - }, - InnerFunc::Avg => AggIntent::Avg { col: None }, - InnerFunc::Min => AggIntent::Min { col: None }, - InnerFunc::Max => AggIntent::Max { col: None }, - InnerFunc::Sum => AggIntent::Sum { col: None }, - InnerFunc::StdDev => AggIntent::StdDev { - col: None, - population: true, - }, - InnerFunc::Variance => AggIntent::Variance { - col: None, - population: true, - }, - InnerFunc::Count => AggIntent::Count { - accuracy: current_accuracy(), - }, - InnerFunc::Rate => AggIntent::Rate, - InnerFunc::IRate => AggIntent::IRate, - InnerFunc::Increase => AggIntent::Increase, - InnerFunc::Changes => AggIntent::Changes, - InnerFunc::Delta => AggIntent::Delta, - InnerFunc::IDelta => AggIntent::IDelta, - InnerFunc::Deriv => AggIntent::Deriv, - InnerFunc::Resets => AggIntent::Resets, - InnerFunc::PredictLinear(s) => AggIntent::PredictLinear { seconds: *s }, - InnerFunc::DoubleExp { smoothing, trend } => AggIntent::DoubleExpSmoothing { - smoothing: *smoothing, - trend: *trend, - }, - InnerFunc::LastOverTime => AggIntent::LastOverTime, - InnerFunc::FirstOverTime => AggIntent::FirstOverTime, - InnerFunc::MadOverTime => AggIntent::MadOverTime, - InnerFunc::TsOfMinOverTime => AggIntent::TsOfMinOverTime, - InnerFunc::TsOfMaxOverTime => AggIntent::TsOfMaxOverTime, - InnerFunc::TsOfFirstOverTime => AggIntent::TsOfFirstOverTime, - InnerFunc::TsOfLastOverTime => AggIntent::TsOfLastOverTime, - } -} - -fn outer_intent(o: &OuterIntent) -> AggIntent { - match o { - OuterIntent::Sum => AggIntent::Sum { col: None }, - OuterIntent::Avg => AggIntent::Avg { col: None }, - OuterIntent::Min => AggIntent::Min { col: None }, - OuterIntent::Max => AggIntent::Max { col: None }, - OuterIntent::StdDev => AggIntent::StdDev { - col: None, - population: true, - }, - OuterIntent::Variance => AggIntent::Variance { - col: None, - population: true, - }, - OuterIntent::Quantile(q) => AggIntent::Quantile { - col: None, - q: *q, - accuracy: current_accuracy(), - }, - OuterIntent::Group => AggIntent::Group, - } -} - -/// Unwrap a (possibly parenthesised) string literal — `count_values` labels and -/// `label_replace`/`label_join` arguments are all string literals, sometimes -/// wrapped in parens (`count_values((("v")), …)`). -fn expr_str(expr: &Expr) -> Result { - match expr { - Expr::StringLiteral(s) => Ok(s.val.clone()), - Expr::Paren(p) => expr_str(&p.expr), - other => Err(LoweringError::InvalidParameter(format!( - "expected a string literal, got `{other}`" - ))), - } -} - -/// A `count_values` string parameter (the synthesized label name). -fn str_param(agg: &AggregateExpr) -> Result { - match &agg.param { - Some(e) => expr_str(e), - None => Err(LoweringError::MissingArgument( - "`count_values` label parameter".into(), - )), - } -} - -/// A call's `idx`-th argument as a string literal (`label_replace`/`label_join`). -fn str_arg(call: &Call, idx: usize) -> Result { - expr_str(arg(call, idx)?) -} - -/// Resolve an aggregation's grouping modifier into a `(keys, without)` pair. -/// -/// `by(labels)` → the kept labels, `without = false`. `without(labels)` → the -/// **excluded** labels, `without = true`: the kept set (the complement) can't be -/// enumerated under an open usage-derived schema, so it is deferred to the -/// runtime and only the excluded positions are carried (issue #39). Both forms -/// canonicalise their label set (sort + dedup) so equivalent groupings lower -/// identically. PromQL labels have no table qualifier → `ColumnRef::Named`. -fn resolve_group(agg: &AggregateExpr) -> Result<(Vec, bool)> { - let canon = |labels: &[String]| -> Vec { - let mut keys = labels.to_vec(); - keys.sort(); - keys.dedup(); - keys.into_iter().map(ColumnRef::Named).collect() - }; - match &agg.modifier { - None => Ok((vec![], false)), - Some(LabelModifier::Include(ls)) => Ok((canon(&ls.labels), false)), - Some(LabelModifier::Exclude(ls)) => Ok((canon(&ls.labels), true)), - } -} - -// ── Free helpers ────────────────────────────────────────────────────────────── - -fn vs_parts(vs: &VectorSelector) -> Result<(String, Vec, TimeShift)> { - // A non-equality `__name__` matcher (`=~` / `!~` / `!=`) selects *across* - // metric names. `Source::TimeSeries { metric }` carries a single concrete - // metric name, so there is no representation for a regex/negated name - // match — reject rather than mislower it to a literal metric named after - // the pattern (issue #67). An equality `__name__` (`{__name__="up"}`) - // still names the metric below. - if let Some(m) = vs - .matchers - .matchers - .iter() - .find(|m| m.name == "__name__" && !matches!(m.op, MatchOp::Equal)) - { - return Err(LoweringError::UnsupportedFeature(format!( - "non-equality `__name__` matcher ({}{:?}) selects across metric names, \ - which has no single-metric canonical representation", - m.name, m.op - ))); - } - let metric = vs.name.clone().unwrap_or_else(|| { - vs.matchers - .matchers - .iter() - .find(|m| m.name == "__name__") - .map(|m| m.value.clone()) - .unwrap_or_default() - }); - // Label matchers are an unordered set: `{a="1",b="2"}` and `{b="2",a="1"}` - // select the same series. Canonicalise by (name, value) so equivalent - // selectors lower to identical predicates. - let mut ms: Vec<&Matcher> = vs - .matchers - .matchers - .iter() - .filter(|m| m.name != "__name__") - .collect(); - ms.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.value.cmp(&b.value))); - let matchers = ms.into_iter().map(matcher_to_compare).collect(); - let shift = time_shift(vs.offset.as_ref(), vs.at.as_ref())?; - Ok((metric, matchers, shift)) -} - -/// Convert the parser's `offset` / `@` modifiers into a [`TimeShift`] (issue -/// #40). Offset is signed milliseconds; `@ ` (parser seconds → ms) becomes -/// an absolute anchor, `@ start()`/`@ end()` the range bounds. -fn time_shift(offset: Option<&Offset>, at: Option<&ParserAtModifier>) -> Result { - let offset_ms = match offset { - None => 0, - Some(Offset::Pos(d)) => duration_ms(*d)?, - Some(Offset::Neg(d)) => -duration_ms(*d)?, - }; - let at = match at { - None => None, - Some(ParserAtModifier::Start) => Some(AtModifier::Start), - Some(ParserAtModifier::End) => Some(AtModifier::End), - Some(ParserAtModifier::At(t)) => Some(AtModifier::Timestamp(system_time_ms(*t)?)), - }; - Ok(TimeShift { offset_ms, at }) -} - -/// A `Duration` as `i64` milliseconds, rejecting an overflow rather than -/// silently truncating a pathologically large `offset`. -fn duration_ms(d: Duration) -> Result { - i64::try_from(d.as_millis()).map_err(|_| { - LoweringError::InvalidParameter("offset duration overflows i64 milliseconds".into()) - }) -} - -/// A `SystemTime` (`@ `) as `i64` milliseconds since the Unix epoch, signed -/// so pre-epoch anchors (the parser permits them) are preserved. -fn system_time_ms(t: SystemTime) -> Result { - let ms = match t.duration_since(std::time::UNIX_EPOCH) { - Ok(d) => i64::try_from(d.as_millis()), - Err(e) => i64::try_from(e.duration().as_millis()).map(|ms| -ms), - }; - ms.map_err(|_| { - LoweringError::InvalidParameter("`@` timestamp overflows i64 milliseconds".into()) - }) -} - -fn matcher_to_compare(m: &Matcher) -> Unresolved { - let op = match &m.op { - MatchOp::Equal => CompareOpKind::Eq, - MatchOp::NotEqual => CompareOpKind::Ne, - MatchOp::Re(_) => CompareOpKind::Regex, - MatchOp::NotRe(_) => CompareOpKind::NotRegex, - }; - Unresolved::Compare { - left: Rc::new(Unresolved::Column(ColumnRef::Named(m.name.clone()))), - op, - right: Rc::new(Unresolved::Literal(ScalarValue::Utf8(m.value.clone()))), - } -} - -fn extract_matrix(expr: &Expr) -> Result<(String, Vec, Duration, TimeShift)> { - match expr { - Expr::MatrixSelector(ms) => { - let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok((metric, matchers, ms.range, shift)) - } - Expr::Paren(p) => extract_matrix(&p.expr), - // A range-vector function argument must be a (parenthesised) matrix - // selector. Do NOT descend through an arbitrary `Call` — that would - // silently strip an unsupported wrapper (`rate(deriv(m[5m]))` lowering - // as `rate(m[5m])`). Reject instead. - other => Err(LoweringError::UnsupportedFeature(format!( - "expected a range-vector (matrix) argument, got `{other}`" - ))), - } -} - -fn arg(call: &Call, idx: usize) -> Result<&Expr> { - call.args - .args - .get(idx) - .map(|b| b.as_ref()) - .ok_or_else(|| LoweringError::MissingArgument(format!("{} arg #{idx}", call.func.name))) -} - -fn num_arg(call: &Call, idx: usize) -> Result { - num_expr(arg(call, idx)?) -} - -fn num_param(agg: &AggregateExpr) -> Result { - match &agg.param { - Some(e) => num_expr(e), - None => Err(LoweringError::MissingArgument( - "aggregate parameter (k / φ)".into(), - )), - } -} - -fn num_expr(expr: &Expr) -> Result { - match expr { - Expr::NumberLiteral(n) => Ok(n.val), - Expr::Paren(p) => num_expr(&p.expr), - // Constant-fold a pure scalar arithmetic expression — the parser does - // not fold `10*1024*1024` / `24 * 3600`. A `modifier` (vector matching) - // or a non-arithmetic operator means it is not a pure scalar. - Expr::Binary(b) if b.modifier.is_none() => { - let (l, r) = (num_expr(&b.lhs)?, num_expr(&b.rhs)?); - let id = b.op.id(); - if id == token::T_ADD { - Ok(l + r) - } else if id == token::T_SUB { - Ok(l - r) - } else if id == token::T_MUL { - Ok(l * r) - } else if id == token::T_DIV { - Ok(l / r) - } else if id == token::T_MOD { - Ok(l % r) - } else if id == token::T_POW { - Ok(l.powf(r)) - } else { - Err(LoweringError::InvalidParameter( - "non-arithmetic operator in scalar expression".into(), - )) - } - } - // `min_of`/`max_of` are n-ary *scalar* reducers (issue #89). Fold them - // when every argument is itself a constant scalar — this is the only - // form the intent algebra can hold (there is no scalar min/max node). A - // non-constant argument (`min_of(step(), 1s)`) fails the recursive fold - // and propagates the error, so it stays rejected. `f64::min`/`max` - // ignore NaN, matching PromQL's `min`/`max` NaN semantics. - Expr::Call(c) if is_scalar_reducer_fn(c.func.name) => { - let reduce = if c.func.name == "min_of" { - f64::min - } else { - f64::max - }; - c.args - .args - .iter() - .map(|a| num_expr(a)) - .reduce(|acc, v| Ok(reduce(acc?, v?))) - .ok_or_else(|| { - LoweringError::MissingArgument(format!("{} needs an argument", c.func.name)) - })? - } - other => Err(LoweringError::InvalidParameter(format!( - "expected a numeric scalar, got `{other}`" - ))), - } -} - -/// The n-ary scalar min/max reducers, foldable when all arguments are constant -/// scalars (issue #89). -fn is_scalar_reducer_fn(name: &str) -> bool { - matches!(name, "min_of" | "max_of") -} - -/// A `BinaryOp` operand: fold a pure-scalar expression (`5`, `10*1024*1024`) to -/// a `PromqlScalarBridge` leaf, otherwise walk it as a vector (issue #35). -fn scalar_or_vector(expr: &Expr) -> Result { - match num_expr(expr) { - Ok(v) => Ok(Unresolved::promql_scalar(v)), - Err(_) => walk(expr), - } -} - -/// `topk`/`bottomk` count parameter — a non-negative integer. Rejects -/// fractional / negative / non-finite values rather than silently truncating -/// or saturating them via `as u64` (`topk(2.7, …)` ≠ `topk(2, …)`). -fn count_param(agg: &AggregateExpr) -> Result { - let v = num_param(agg)?; - if v.is_finite() && v >= 0.0 && v.fract() == 0.0 && v <= u64::MAX as f64 { - Ok(v as u64) - } else { - Err(LoweringError::InvalidParameter(format!( - "topk/bottomk k must be a non-negative integer, got {v}" - ))) - } -} - -/// `limit_ratio` ratio parameter — a finite value; Prometheus clamps it to -/// `[-1, 1]` (a negative ratio selects the complementary fraction). A non-finite -/// ratio (`limit_ratio(NaN, …)`) or a dynamic one (`time() % 17/17`, which -/// `num_param` can't fold) is rejected (issue #86). -fn ratio_param(agg: &AggregateExpr) -> Result { - let r = num_param(agg)?; - if !r.is_finite() { - return Err(LoweringError::InvalidParameter(format!( - "limit_ratio ratio must be finite, got {r}" - ))); - } - Ok(r.clamp(-1.0, 1.0)) -} - -/// Preserve the full Prometheus quantile parameter domain, including special values. -fn quantile_param(q: f64) -> Result { - // Prometheus returns NaN/-Inf/+Inf for these parameters at execution time. - Ok(q) -} - -// The non-standard histogram_quantiles extension keeps its bounded label contract. -fn bounded_quantile_param(q: f64) -> Result { - if q.is_finite() && (0.0..=1.0).contains(&q) { - Ok(q) - } else { - Err(LoweringError::InvalidParameter(format!( - "quantile φ must be in [0, 1], got {q}" - ))) - } -} - -fn binop(id: token::TokenId) -> Result { - Ok(if id == token::T_ADD { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Add) - } else if id == token::T_SUB { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub) - } else if id == token::T_MUL { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul) - } else if id == token::T_DIV { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Div) - } else if id == token::T_MOD { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Mod) - } else if id == token::T_POW { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Pow) - } else if id == token::T_ATAN2 { - BinaryOpKind::Arithmetic(ArithmeticOpKind::Atan2) - } else if id == token::T_EQLC { - BinaryOpKind::Compare(CompareOpKind::Eq) - } else if id == token::T_NEQ { - BinaryOpKind::Compare(CompareOpKind::Ne) - } else if id == token::T_LSS { - BinaryOpKind::Compare(CompareOpKind::Lt) - } else if id == token::T_LTE { - BinaryOpKind::Compare(CompareOpKind::Le) - } else if id == token::T_GTR { - BinaryOpKind::Compare(CompareOpKind::Gt) - } else if id == token::T_GTE { - BinaryOpKind::Compare(CompareOpKind::Ge) - } else if id == token::T_LAND { - BinaryOpKind::Set(PromQLVectorSetOpKind::And) - } else if id == token::T_LOR { - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) - } else if id == token::T_LUNLESS { - BinaryOpKind::Set(PromQLVectorSetOpKind::Unless) - } else { - return Err(LoweringError::UnsupportedFeature(format!( - "binary operator token {id}" - ))); - }) -} diff --git a/crates/frontend-promql/tests/histogram_metadata.rs b/crates/frontend-promql/tests/histogram_metadata.rs deleted file mode 100644 index 55f35ddeb..000000000 --- a/crates/frontend-promql/tests/histogram_metadata.rs +++ /dev/null @@ -1,140 +0,0 @@ -//! Type-driven `histogram_quantile` discrimination (issue #79). -//! -//! The structural heuristic (`by (le)` / `_bucket` / `le=` matcher) proxies the -//! argument's sample type. A declared [`HistogramKind`] overrides it, fixing the -//! heuristic's false-positive and false-negative cases. Undeclared metrics still -//! fall back to the heuristic. - -use asap_frontend_promql::{HistogramCatalog, HistogramKind}; -mod support; -use asap_types::pre_asap::{AggIntent, QueryExpr}; -use asap_types::types::AccuracyTarget; -use support::{lower_promql, lower_promql_with_histograms}; - -/// The histogram/quantile intent kind in the lowered DAG: `"HQ"` for the -/// classic-bucket `HistogramQuantile`, `"Q"` for the sketch-able `Quantile`. -fn quantile_kind(qe: &QueryExpr) -> &'static str { - fn walk(e: &QueryExpr) -> Option<&'static str> { - match e { - QueryExpr::Aggregate { - measures, child, .. - } => measures - .iter() - .find_map(|i| match i { - AggIntent::HistogramQuantile { .. } => Some("HQ"), - AggIntent::Quantile { .. } => Some("Q"), - _ => None, - }) - .or_else(|| walk(child)), - QueryExpr::TimeRange { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::Project { child, .. } => walk(child), - _ => None, - } - } - walk(qe).expect("a HistogramQuantile or Quantile intent") -} - -fn heuristic(q: &str) -> &'static str { - quantile_kind(&lower_promql(q, AccuracyTarget::Exact).unwrap()) -} - -fn with_meta(q: &str, catalog: HistogramCatalog) -> &'static str { - quantile_kind(&lower_promql_with_histograms(q, AccuracyTarget::Exact, catalog).unwrap()) -} - -#[test] -fn heuristic_baseline_is_unchanged_without_a_catalog() { - // Classic `by (le)`-bucket form → HistogramQuantile; anything else → Quantile. - assert_eq!( - heuristic( - "histogram_quantile(0.9, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))" - ), - "HQ" - ); - assert_eq!(heuristic("histogram_quantile(0.9, native_latency)"), "Q"); -} - -#[test] -fn declared_classic_bucket_fixes_the_false_negative() { - // A classic histogram exposed WITHOUT the `_bucket` suffix and queried with - // no `le` grouping/matcher: the heuristic wrongly routes it to the - // sketch-able Quantile. Declaring it `ClassicBucket` corrects it. - let q = "histogram_quantile(0.9, latency_seconds)"; - assert_eq!( - heuristic(q), - "Q", - "heuristic mis-routes the suffix-less classic histogram" - ); - assert_eq!( - with_meta( - q, - HistogramCatalog::new().with("latency_seconds", HistogramKind::ClassicBucket) - ), - "HQ", - "metadata routes it to exact bucket interpolation" - ); -} - -#[test] -fn declared_raw_or_native_fixes_the_false_positive() { - // A metric merely NAMED `…_bucket` that actually holds raw samples / a native - // histogram: the heuristic wrongly routes it to bucket interpolation. - let q = "histogram_quantile(0.9, foo_bucket)"; - assert_eq!( - heuristic(q), - "HQ", - "heuristic mis-routes on the `_bucket` name" - ); - assert_eq!( - with_meta( - q, - HistogramCatalog::new().with("foo_bucket", HistogramKind::RawSamples) - ), - "Q", - "raw samples are sketch-able" - ); - assert_eq!( - with_meta( - q, - HistogramCatalog::new().with("foo_bucket", HistogramKind::Native) - ), - "Q", - "native histograms are sketch-able" - ); -} - -#[test] -fn undeclared_metric_falls_back_to_the_heuristic() { - // A catalog that doesn't mention the queried metric leaves the structural - // decision in place. - let catalog = HistogramCatalog::new().with("some_other_metric", HistogramKind::RawSamples); - assert_eq!( - with_meta( - "histogram_quantile(0.9, sum by (le) (x_bucket))", - catalog.clone() - ), - "HQ" - ); - assert_eq!( - with_meta("histogram_quantile(0.9, native_thing)", catalog), - "Q" - ); -} - -#[test] -fn the_catalog_does_not_leak_across_calls() { - // The ambient catalog is scoped to the single `_with_histograms` call; a - // subsequent plain `lower_promql` sees no metadata (guards against a - // thread-local that isn't cleaned up). - let _ = with_meta( - "histogram_quantile(0.9, foo_bucket)", - HistogramCatalog::new().with("foo_bucket", HistogramKind::RawSamples), - ); - // `foo_bucket` would be sketch-able under that catalog, but with none it must - // revert to the heuristic (the `_bucket` name → HistogramQuantile). - assert_eq!(heuristic("histogram_quantile(0.9, foo_bucket)"), "HQ"); -} diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs deleted file mode 100644 index 6f932879e..000000000 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ /dev/null @@ -1,2291 +0,0 @@ -//! PromQL **semantic conformance** for the parse-to-canonical-DAG lowering. -//! -//! We *lower* PromQL to the intent algebra; we do not *execute* it. So "same -//! semantic job as Prometheus" here means: for each canonical query, does the -//! canonical DAG encode the **documented PromQL meaning** — and where we knowingly -//! diverge (reject, approximate, or drop a modifier), is that pinned by a test -//! so it stays visible? -//! -//! Sources for the queries + their semantics: -//! - PromQL basics (data types, selectors, offset/@/subquery): -//! -//! - PromLabs PromQL cheat sheet (common real-world queries by category): -//! -//! - Prometheus' own engine test corpus (these are *execution* tests — -//! load → eval → expect values — so they define semantics we mirror as -//! *structure*): -//! Relevant files, mapped to the sections below: selectors.test, -//! aggregators.test, functions.test, histograms.test, operators.test, -//! subquery.test, at_modifier.test, literals.test, limit.test -//! -//! Legend used in test names: -//! - (no suffix) — we lower it and the canonical intent matches PromQL. -//! - `__GAP` — a PromQL capability we don't *yet* support. It is **cleanly -//! rejected** (never silently mislowered), and pinned here so adding support -//! later flips the assertion deliberately. -//! -//! NOTE: the formerly-silent divergences (`group`→sum, dropped `offset`/`@`, -//! `changes`/`resets`→count) are now rejected rather than mislowered — see the -//! equivalence suite (`promql_equivalence.rs`) and section L below. - -// `__GAP`-suffixed test names intentionally SHOUT the documented divergences. -#![allow(non_snake_case)] - -use std::time::Duration; - -use asap_frontend_promql::PromqlError as LoweringError; -mod support; -use asap_types::pre_asap::schema::DataType; -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, AtModifier, BinaryOpKind, CompareOpKind, MathFunc, - PromQLVectorSetOpKind, QueryExpr, Reduction, SampleKind, Source, TimeFunc, -}; -use asap_types::types::AccuracyTarget; -use support::lower_promql; - -// ── harness helpers ───────────────────────────────────────────────────────────── - -/// Lower, expecting success. -fn ok(q: &str) -> QueryExpr { - lower_promql(q, AccuracyTarget::Exact) - .unwrap_or_else(|e| panic!("expected {q:?} to lower, got error: {e}")) -} - -/// Lower, expecting a clean `LoweringError` (an unsupported capability). -fn rejected(q: &str) -> LoweringError { - match lower_promql(q, AccuracyTarget::Exact) { - Err(e) => e, - Ok(dag) => panic!("expected {q:?} to be rejected, but it lowered to: {dag:?}"), - } -} - -/// Every `AggIntent` anywhere in the DAG, root-to-leaf. -fn intents(e: &QueryExpr) -> Vec { - let mut out = Vec::new(); - collect(e, &mut out); - out -} - -fn collect(e: &QueryExpr, out: &mut Vec) { - match e { - QueryExpr::Aggregate { - measures, child, .. - } => { - out.extend(measures.iter().cloned()); - collect(child, out); - } - QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::PromqlInfoEnrich { child, .. } => collect(child, out), - QueryExpr::BinaryOp { lhs, rhs, .. } => { - collect(lhs, out); - collect(rhs, out); - } - QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } => { - collect(left, out); - collect(right, out); - } - QueryExpr::Concat { children, .. } => children.iter().for_each(|c| collect(c, out)), - QueryExpr::PromqlVectorFromScalar(inner) | QueryExpr::PromqlScalarFromVector(inner) => { - collect(inner, out) - } - // `AggIntent` only ever lives in `Aggregate.measures`, never in a - // scalar position (issue #205) — nothing to collect there. - QueryExpr::Scan { .. } - | QueryExpr::PromqlScalarBridge(_) - | QueryExpr::EvalTimestamp - | QueryExpr::CurrentTimestamp => {} - QueryExpr::Column(_) - | QueryExpr::Literal(_) - | QueryExpr::Compare { .. } - | QueryExpr::BoolAnd(_) - | QueryExpr::BoolOr(_) - | QueryExpr::Not(_) - | QueryExpr::IsNull(_) - | QueryExpr::IsNotNull(_) - | QueryExpr::Cast { .. } - | QueryExpr::InList { .. } - | QueryExpr::FunctionCall { .. } - | QueryExpr::Arithmetic { .. } - | QueryExpr::Case { .. } => {} - } -} - -/// The first `Scan` reached by descending single-child nodes, with its metric -/// name and predicate count. -fn first_scan(e: &QueryExpr) -> (String, usize) { - match e { - QueryExpr::Scan { - source, predicates, .. - } => { - let name = match source { - Source::TimeSeries { metric } => metric.clone(), - Source::Table { table_ref } => table_ref.clone(), - }; - (name, predicates.len()) - } - QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => first_scan(child), - other => panic!("no Scan reachable from {other:?}"), - } -} - -fn has bool>(e: &QueryExpr, pred: F) -> bool { - intents(e).iter().any(pred) -} - -/// Whether the DAG contains a `Mul`-by-`PromqlScalarBridge(-1)` anywhere — the shape unary -/// negation lowers to (issue #36). -fn negates_via_scalar(e: &QueryExpr) -> bool { - let is_neg_one = |q: &QueryExpr| { - q.as_promql_scalar() - .is_some_and(|v| (v + 1.0).abs() < 1e-12) - }; - match e { - QueryExpr::BinaryOp { op, lhs, rhs, .. } => { - (*op == BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul) - && (is_neg_one(lhs) || is_neg_one(rhs))) - || negates_via_scalar(lhs) - || negates_via_scalar(rhs) - } - QueryExpr::Aggregate { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } => negates_via_scalar(child), - _ => false, - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// A. Selectors & label matchers (basics §"Instant/Range Vector -// Selectors"; selectors.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn instant_vector_selector() { - // SEMANTICS: bare metric → instant vector (latest sample per series). - let (metric, preds) = first_scan(&ok("node_cpu_seconds_total")); - assert_eq!(metric, "node_cpu_seconds_total"); - assert_eq!(preds, 0, "no label matchers → no predicates"); -} - -#[test] -fn promql_scan_schema_is_open() { - // A schemaless PromQL leaf is *open*: the metric's full label set is - // runtime-only, so the binding schema lists only the (ts, value) floor + - // referenced labels and may be a subset of the runtime row. - let qe = ok("node_cpu_seconds_total"); - let QueryExpr::TimeRange { child, .. } = &qe else { - panic!("expected a TimeRange for a bare selector, got {qe:?}"); - }; - let QueryExpr::Scan { schema, .. } = child.as_ref() else { - panic!("expected a Scan inside the TimeRange, got {qe:?}"); - }; - assert!( - !schema.closed, - "a schemaless PromQL scan has an open schema" - ); -} - -#[test] -fn label_matchers_become_scan_predicates() { - // SEMANTICS: `=`, `!=`, `=~`, `!~` filter series; one conjunct per matcher. - let (_, preds) = first_scan(&ok( - r#"http_requests_total{job!="x",path=~"/api/.*",env!~"dev"}"#, - )); - assert_eq!(preds, 3, "three matchers → three Scan predicates"); -} - -#[test] -fn name_label_selects_the_metric() { - // SEMANTICS: the metric name is the internal `__name__` label. - let (metric, preds) = first_scan(&ok(r#"{__name__="up"}"#)); - assert_eq!(metric, "up"); - assert_eq!(preds, 0, "__name__ is the metric, not a residual predicate"); -} - -#[test] -fn name_regex_matcher_is_rejected__GAP() { - // A `__name__=~` / `!~` / `!=` matcher selects *across* metric names, which - // the single-metric `Source::TimeSeries { metric }` can't represent. It is - // rejected (issue #67) rather than silently mislowered to a literal metric - // named after the pattern (`{__name__=~"node_.*"}` → `Source("node_.*")`). - // Full support needs a wildcard/regex `Source` in the IR. - let _ = rejected(r#"{__name__=~"node_.*"}"#); - let _ = rejected(r#"{__name__!~"x", job="y"}"#); - // Equality still names the metric (regression guard for the fix). - let (metric, _) = first_scan(&ok(r#"{__name__="up"}"#)); - assert_eq!(metric, "up"); -} - -#[test] -fn range_vector_selector_is_time_range() { - // SEMANTICS: `[5m]` turns an instant vector into a range vector, - // represented in the canonical DAG as a dedicated `TimeRange` node. - let qe = ok("node_cpu_seconds_total[5m]"); - let QueryExpr::TimeRange { range, .. } = &qe else { - panic!("expected TimeRange for a range-vector selector, got {qe:?}"); - }; - assert_eq!(*range, Duration::from_secs(300)); -} - -// ───────────────────────────────────────────────────────────────────────────── -// B. Counters: rate / irate / increase (cheat sheet "Rates of Increase"; -// functions.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn rate_range_lives_in_time_range_node() { - // SEMANTICS: per-second average rate; the temporal range lives on the - // enclosing `TimeRange` node, not inside the intent. - let qe = ok("rate(http_requests_total[5m])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Rate])); - let QueryExpr::TimeRange { range, .. } = child.as_ref() else { - panic!("expected TimeRange child, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(300)); -} - -#[test] -fn irate_maps_to_its_own_intent() { - assert!(has(&ok("irate(http_requests_total[1m])"), |i| matches!( - i, - AggIntent::IRate - ))); -} - -#[test] -fn increase_range_lives_in_time_range_node() { - let qe = ok("increase(http_requests_total[1h])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Increase])); - let QueryExpr::TimeRange { range, .. } = child.as_ref() else { - panic!("expected TimeRange child, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(3600)); -} - -// ───────────────────────────────────────────────────────────────────────────── -// C. Aggregation across series (cheat sheet "Aggregating Over -// Multiple Series"; aggregators.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn sum_collapses_all_series() { - // SEMANTICS: `sum(v)` → one output series. No grouping → no Partition. - let qe = ok("sum(node_filesystem_size_bytes)"); - assert!(matches!(&qe, QueryExpr::Aggregate { .. })); - assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); -} - -#[test] -fn sum_by_groups_via_positional_aggregate() { - // SEMANTICS: `by(job,instance)` keeps those labels; the grouping lives on a - // positional `Aggregate.by` — the same shape SQL `GROUP BY` produces (not a - // name-based Partition). SchemaResolver leaf = [ts, value, instance, job] (referenced - // keys appended sorted), so the keys resolve to columns [2, 3]. - let qe = ok("sum by(job, instance) (node_filesystem_size_bytes)"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected positional Aggregate for `by(...)`, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::by(vec![2, 3]), - "group keys resolve to positional ColumnIds" - ); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Scan { .. })) - ); -} - -#[test] -fn count_is_row_count() { - assert!(has(&ok("count(up)"), |i| matches!( - i, - AggIntent::Count { .. } - ))); -} - -#[test] -fn avg_min_max_stddev_stdvar_quantile_aggregators() { - assert!(has(&ok("avg(up)"), |i| matches!(i, AggIntent::Avg { .. }))); - assert!(has(&ok("min(up)"), |i| matches!(i, AggIntent::Min { .. }))); - assert!(has(&ok("max(up)"), |i| matches!(i, AggIntent::Max { .. }))); - assert!(has(&ok("stddev(up)"), |i| matches!( - i, - AggIntent::StdDev { .. } - ))); - assert!(has(&ok("stdvar(up)"), |i| matches!( - i, - AggIntent::Variance { .. } - ))); - assert!(has(&ok("quantile(0.5, up)"), |i| matches!( - i, - AggIntent::Quantile { .. } - ))); -} - -#[test] -fn sum_without_groups_by_the_complement() { - // SEMANTICS (issue #39): `without(instance)` = group by all labels EXCEPT - // instance. The complement can't be enumerated under the open usage-derived - // schema, so the excluded label is stored and the kept set is deferred to - // the runtime: the grouping is the exclusion form and the output schema - // stays OPEN (unlike `by`, which freezes to closed). - let qe = ok("sum without(instance) (node_filesystem_size_bytes)"); - let QueryExpr::Aggregate { - reduction, - measures, - .. - } = &qe - else { - panic!("expected an Aggregate, got {qe:?}"); - }; - let by = reduction.expect_reduce(); - assert!( - by.is_without(), - "the grouping is the `without` exclusion form" - ); - assert_eq!(by.keys().len(), 1, "the one excluded label (instance)"); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!( - !qe.output_schema().unwrap().closed, - "a `without` result keeps an open schema (kept label set is runtime-only)" - ); -} - -#[test] -fn without_on_topk_is_rejected() { - // `without(...)` is modelled only for reducing aggregations; on topk/bottomk - // (a ranking, not a reduction) it would need without-partitioning, so it is - // rejected rather than silently lowered as a `by` (issue #39). - let e = rejected("topk without (job) (3, http_requests_total)"); - assert!(format!("{e}").contains("without"), "got {e}"); -} - -#[test] -fn group_aggregator_lowers_to_a_distinct_intent() { - // SEMANTICS (PromQL): `group(v)` returns a constant 1 per group (presence), - // NOT a sum. It now lowers to a distinct `Group` intent (never folded onto - // `Sum`) — see §S. Regression guard that it is not a `Sum`. - let qe = ok("group by (job) (up)"); - assert!(has(&qe, |i| *i == AggIntent::Group)); - assert!(!has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); -} - -// ───────────────────────────────────────────────────────────────────────────── -// D. Two-level: outer aggregation OVER an inner counter (the canonical -// `sum(rate(...))` shape; aggregators.test + functions.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn sum_of_rate_is_two_levels() { - // SEMANTICS: per-series rate, THEN cross-series sum. Both must survive. - let qe = ok("sum(rate(http_requests_total[5m]))"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!(matches!( - child.as_ref(), - QueryExpr::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate]) - )); -} - -#[test] -fn sum_by_of_rate_groups_outer_level() { - // Outer cross-series Sum grouped on positional `Aggregate.by` over the - // label-preserving inner Rate. Leaf = [ts, value, instance] → by = [2]. - let qe = ok("sum by(instance) (rate(node_network_receive_bytes_total[5m]))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate grouped by instance, got {qe:?}"); - }; - assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - // child is the inner per-series Rate aggregate. - assert!(matches!( - child.as_ref(), - QueryExpr::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate]) - )); -} - -#[test] -fn sum_by_of_over_time_groups_outer_level() { - // Outer cross-series Sum grouped on positional `Aggregate.by` over an inner - // *per-series* `avg_over_time` — `Window { Aggregate{Avg} }` is label- - // preserving, so the key resolves positionally just like the rate case (no - // name-based Partition). Leaf = [ts, value, instance] → by = [2]. - let qe = ok("sum by(instance) (avg_over_time(node_cpu_seconds_total[5m]))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate grouped by instance, got {qe:?}"); - }; - assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - // child is the inner per-series reduction: Aggregate{Avg} over TimeRange. - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected Aggregate (per-series avg_over_time) under the Sum, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Avg { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -// ───────────────────────────────────────────────────────────────────────────── -// E. Aggregation over time (per-series) (cheat sheet "Aggregating Over -// Time"; functions.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn over_time_functions_reduce_over_time_range() { - // SEMANTICS: reduce the samples WITHIN each series over the range → - // Aggregate over TimeRange (per-series, label-preserving). - for (q, want) in [ - ("avg_over_time(go_goroutines[5m])", "avg"), - ("max_over_time(process_resident_memory_bytes[1d])", "max"), - ("min_over_time(go_goroutines[5m])", "min"), - ("sum_over_time(go_goroutines[5m])", "sum"), - ("count_over_time(go_goroutines[5m])", "count"), - ] { - let qe = ok(q); - assert!( - matches!(&qe, QueryExpr::Aggregate { .. }), - "{q}: expected Aggregate" - ); - let matched = intents(&qe).iter().any(|i| match want { - "avg" => matches!(i, AggIntent::Avg { .. }), - "max" => matches!(i, AggIntent::Max { .. }), - "min" => matches!(i, AggIntent::Min { .. }), - "sum" => matches!(i, AggIntent::Sum { .. }), - "count" => matches!(i, AggIntent::Count { .. }), - _ => unreachable!(), - }); - assert!(matched, "{q}: missing {want} intent"); - } -} - -#[test] -fn quantile_over_time_is_aggregate_over_time_range() { - let qe = ok("quantile_over_time(0.9, request_latency_seconds[5m])"); - assert!(matches!(&qe, QueryExpr::Aggregate { .. })); - assert!(has( - &qe, - |i| matches!(i, AggIntent::Quantile { q, .. } if (*q - 0.9).abs() < 1e-9) - )); -} - -// ───────────────────────────────────────────────────────────────────────────── -// F. Histograms (cheat sheet "Quantiles from -// Histograms"; histograms.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn histogram_quantile_over_rate() { - // φ-quantile from bucket rates. The `_bucket` metric marks the classic - // cumulative-bucket form → `HistogramQuantile` (even without `sum by (le)`). - let qe = ok("histogram_quantile(0.9, rate(demo_api_request_duration_seconds_bucket[5m]))"); - let QueryExpr::Aggregate { measures, .. } = &qe else { - panic!("expected Aggregate{{HistogramQuantile}}, got {qe:?}"); - }; - assert!( - matches!(measures.as_slice(), [AggIntent::HistogramQuantile { q, .. }] if (*q - 0.9).abs() < 1e-9) - ); - assert!(has(&qe, |i| matches!(i, AggIntent::Rate))); -} - -#[test] -fn histogram_quantile_over_sum_by_le_preserves_le_grouping() { - // SEMANTICS: the standard pattern — bucket rates summed by `le`, then the - // quantile. The `sum by (le)` aggregation must survive into the - // canonical DAG. - let qe = ok( - "histogram_quantile(0.99, sum by(le) (rate(demo_api_request_duration_seconds_bucket[5m])))", - ); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate{{HistogramQuantile}}, got {qe:?}"); - }; - // `by (le)` marks the classic cumulative-bucket form → `HistogramQuantile`. - assert!(matches!( - measures.as_slice(), - [AggIntent::HistogramQuantile { .. }] - )); - // `sum by(le)` now survives as a positional Aggregate (by = [2], `le`), over - // the inner Rate — no name-based Partition. - let QueryExpr::Aggregate { - reduction, - measures, - .. - } = child.as_ref() - else { - panic!("expected `sum by(le)` as a positional Aggregate, got {child:?}"); - }; - assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); -} - -// ───────────────────────────────────────────────────────────────────────────── -// G. Binary ops: math, matching, comparison (cheat sheet "Math Between -// Series" / "Filtering Series by Value"; operators.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn vector_arithmetic() { - let qe = ok("node_memory_MemFree_bytes + node_memory_Cached_bytes"); - let QueryExpr::BinaryOp { op, .. } = &qe else { - panic!("expected BinaryOp, got {qe:?}"); - }; - assert_eq!(*op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Add)); -} - -#[test] -fn on_matching_with_group_left() { - // SEMANTICS: many-to-one matching on a label subset. - let qe = - ok("rate(demo_cpu_usage_seconds_total[1m]) / on(instance, job) group_left demo_num_cpus"); - let QueryExpr::BinaryOp { - op, vector_match, .. - } = &qe - else { - panic!("expected BinaryOp, got {qe:?}"); - }; - assert_eq!(*op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)); - let vm = vector_match.as_ref().expect("on(...) group_left present"); - assert_eq!(vm.labels, vec!["instance".to_string(), "job".to_string()]); - assert!( - vm.grouping.is_some(), - "group_left should set the grouping side" - ); -} - -#[test] -fn vector_comparison_filters() { - // SEMANTICS: `>` between two vectors keeps the LHS series where it holds. - let qe = ok("go_goroutines > go_threads"); - assert!( - matches!(&qe, QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Compare(CompareOpKind::Gt)) - ); -} - -#[test] -fn unary_negation_lowers_as_multiply_by_minus_one() { - // SEMANTICS (PromQL, issue #36): `-expr` flips the sign of every sample. - // Now that a scalar operand exists (#35), it lowers as `expr * -1` — a `Mul` - // BinaryOp of the (label-preserving) vector against `PromqlScalarBridge(-1)`. These are - // the five cases the old `__GAP` test pinned as rejected. - for q in [ - "-rate(http_errors_total[5m])", - "-some_metric", - "-metric_a or -metric_b", - "http_requests_total - -http_errors_total", - "sum(-node_cpu_seconds_total)", - ] { - let qe = ok(q); - // A `Mul`-by-`-1` against a `PromqlScalarBridge(-1)` appears somewhere in every DAG. - assert!( - negates_via_scalar(&qe), - "no `* -1` negation found in {q}: {qe:?}" - ); - } - - // `-some_metric` at the root: `Scan * PromqlScalarBridge(-1)`, schema follows the vector. - let QueryExpr::BinaryOp { - op, - lhs, - rhs, - vector_match, - } = &ok("-some_metric") - else { - panic!("expected a BinaryOp for `-some_metric`"); - }; - assert_eq!(*op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul)); - assert!( - matches!(lhs.as_ref(), QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Scan { .. })), - "vector on the left" - ); - assert!( - rhs.as_promql_scalar() - .is_some_and(|v| (v + 1.0).abs() < 1e-12), - "negation multiplies by PromqlScalarBridge(-1), got {rhs:?}" - ); - assert!( - vector_match.is_none(), - "scalar negation carries no vector match" - ); - // Label-preserving: the schema is the vector operand's, unchanged. - let schema = ok("-some_metric").output_schema().unwrap(); - assert_eq!( - schema - .fields - .iter() - .map(|c| c.name.as_str()) - .collect::>(), - vec!["ts", "value"], - ); - - // `sum(-m)` — the negation lowers inside the aggregate argument (issue #27 - // nesting), so the outer node is the `Sum` aggregate over the `Mul`. - let QueryExpr::Aggregate { - measures, child, .. - } = &ok("sum(-node_cpu_seconds_total)") - else { - panic!("expected an outer Aggregate for `sum(-m)`"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!(matches!( - child.as_ref(), - QueryExpr::BinaryOp { - op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - .. - } - )); -} - -#[test] -fn unary_negation_of_constant_folds_to_scalar() { - // `-(10*1024*1024)` — the operand is constant-foldable, so negation collapses - // to a single negated `PromqlScalarBridge` leaf (no `BinaryOp`), just like a bare literal. - assert!(ok("-(10*1024*1024)") - .as_promql_scalar() - .is_some_and(|v| (v + 10_485_760.0).abs() < 1e-6)); -} - -#[test] -fn double_unary_negation_nests() { - // `- -some_metric` — negation of a negation: `(m * -1) * -1`. Both levels - // lower; the value is unchanged but the structure is faithfully nested. - let QueryExpr::BinaryOp { op, lhs, .. } = &ok("- -some_metric") else { - panic!("expected outer BinaryOp for `- -some_metric`"); - }; - assert_eq!(*op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul)); - assert!( - matches!( - lhs.as_ref(), - QueryExpr::BinaryOp { - op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), - .. - } - ), - "inner negation nests under the outer one" - ); -} - -#[test] -fn count_maps_to_count_and_inherits_accuracy() { - // Counts preserve the workload accuracy target without counting distinct values. - let exact = lower_promql("count by (job) (up)", AccuracyTarget::Exact).unwrap(); - assert!( - has(&exact, |i| matches!( - i, - AggIntent::Count { - accuracy: AccuracyTarget::Exact - } - )), - "Count must stay Exact under AccuracyTarget::Exact, got {:?}", - intents(&exact) - ); - - let approx = lower_promql("count by (job) (up)", AccuracyTarget::Epsilon(0.01)).unwrap(); - assert!( - has(&approx, |i| matches!( - i, - AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(e) - } if (*e - 0.01).abs() < 1e-9 - )), - "Count must carry the approximate target, got {:?}", - intents(&approx) - ); -} - -#[test] -fn scalar_literal_operand_lowers_as_binaryop_scalar() { - // Issue #35: ` op ` — the numeric threshold is a - // `PromqlScalarBridge` operand of the `BinaryOp`, and constant arithmetic - // (`10*1024*1024`) is folded. The output schema is the vector side's. - let qe = ok("node_filesystem_avail_bytes > 10*1024*1024"); - let QueryExpr::BinaryOp { op, lhs, rhs, .. } = &qe else { - panic!("expected a BinaryOp, got {qe:?}"); - }; - assert_eq!(*op, BinaryOpKind::Compare(CompareOpKind::Gt)); - assert!( - matches!(lhs.as_ref(), QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Scan { .. })), - "vector on the left" - ); - assert!( - rhs.as_promql_scalar() - .is_some_and(|v| (v - 10_485_760.0).abs() < 1e-6), - "folded scalar threshold on the right, got {rhs:?}" - ); - // Schema derivation follows the vector side (a scalar contributes no labels). - assert!(qe.output_schema().is_ok()); -} - -#[test] -fn scalar_arithmetic_scales_the_vector() { - // `rate(m[5m]) * 100` — a unit conversion. Arithmetic BinaryOp of the vector - // with a `PromqlScalarBridge(100)`. - let qe = ok("rate(m[5m]) * 100"); - let QueryExpr::BinaryOp { op, rhs, .. } = &qe else { - panic!("expected a BinaryOp, got {qe:?}"); - }; - assert_eq!(*op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul)); - assert!(rhs - .as_promql_scalar() - .is_some_and(|v| (v - 100.0).abs() < 1e-9)); -} - -// ───────────────────────────────────────────────────────────────────────────── -// H. Set operations (cheat sheet "Set Operations"; -// operators.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn set_ops_lower_to_binaryop() { - // SEMANTICS: or = union of label sets; and = intersection; unless = difference. - assert!(matches!(&ok("up{job=\"a\"} or up{job=\"b\"}"), - QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Set(PromQLVectorSetOpKind::Or))); - assert!(matches!(&ok("node_network_mtu_bytes and node_up"), - QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Set(PromQLVectorSetOpKind::And))); - assert!(matches!(&ok("node_network_mtu_bytes unless node_down"), - QueryExpr::BinaryOp { op, .. } if *op == BinaryOpKind::Set(PromQLVectorSetOpKind::Unless))); -} - -// ───────────────────────────────────────────────────────────────────────────── -// I. Sorting / top-k (cheat sheet "Sorting"/topk; -// functions.test, limit.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn topk_over_count_is_heavy_hitter() { - // SEMANTICS: top-k by frequency → first-class heavy-hitter `TopK` intent. - let qe = ok("topk(10, count_over_time(http_requests_total[1m]))"); - assert!(has( - &qe, - |i| matches!(i, AggIntent::TopK { k, .. } if *k == 10) - )); -} - -#[test] -fn bottomk_is_generic_sort_limit() { - // SEMANTICS: bottom-k → generic ascending order + limit (no sketch). - let qe = ok("bottomk(3, count_over_time(http_requests_total[5m]))"); - assert!(matches!(&qe, QueryExpr::Limit { .. })); -} - -#[test] -fn topk_over_nested_sum_preserves_weighted_topk_accuracy() { - // SEMANTICS (PromQL): `topk(3, sum by(x)(rate(...)))` is extremely common. - // The final rates are query-time values. Their ordering does not establish - // frequency-sketch membership semantics. - let qe = ok("topk(3, sum by(instance) (rate(node_cpu_seconds_total[5m])))"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected weighted TopK aggregate, got {qe:?}"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 3, .. }] - )); - // The inner `sum by (instance)` survives as a cross-series Aggregate over the - // per-series rate — the nesting the old two-level template could not express. - assert!( - has(child, |i| matches!(i, AggIntent::Sum { .. })) - && has(child, |i| matches!(i, AggIntent::Rate)), - "inner sum-over-rate preserved, got {:?}", - intents(child) - ); - assert!(has(&qe, |i| matches!(i, AggIntent::TopK { .. }))); -} - -#[test] -fn outer_aggregate_over_nested_aggregate_nests() { - // `max(sum by (job) (rate(m[5m])))` — an outer cross-series reduction over a - // nested per-group reduction over a per-series rate: three stacked levels the - // flat two-level template rejected. Each level survives into the - // canonical DAG (issue #27). - let qe = ok("max(sum by (job) (rate(http_requests_total[5m])))"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Max { .. }])); - let QueryExpr::Aggregate { - reduction, - measures, - .. - } = child.as_ref() - else { - panic!("expected inner `sum by (job)` Aggregate, got {child:?}"); - }; - assert_eq!( - reduction, - &Reduction::by(vec![2]), - "job grouping survives on the inner aggregate" - ); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!(has(&qe, |i| matches!(i, AggIntent::Rate)), "rate preserved"); -} - -#[test] -fn outer_group_key_absent_from_nested_aggregate_is_dropped() { - // SEMANTICS (PromQL, issue #53): aggregating `by` a label that no input - // series carries is valid — every series lands in one group and the - // (empty) label is omitted from the output. Here the inner `sum by (group)` - // collapses `job` away (its closed output schema is `[group, sum]`), so the - // outer `by (job)` groups everything into a single global partition: - // the query lowers with the provably-absent key dropped, exactly - // `sum(sum by (group)(…))`. - let qe = ok(r#"sum(sum by (group)(http_requests{job="api-server"})) by (job)"#); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::by(vec![]), - "absent `job` key dropped → global aggregate" - ); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - let QueryExpr::Aggregate { reduction, .. } = child.as_ref() else { - panic!("expected inner `sum by (group)` Aggregate, got {child:?}"); - }; - assert_eq!( - reduction, - &Reduction::by(vec![2]), - "inner grouping on `group` survives" - ); -} - -#[test] -fn outer_group_key_present_after_inner_aggregate_still_resolves() { - // The counterpart guard for #53: when the outer key IS in the inner - // aggregate's output (`by (job)` over `sum by (job, group)`), it must keep - // resolving positionally — the absent-key drop only fires on provable - // absence, never on a resolvable key. - let qe = ok("sum(sum by (job, group)(http_requests)) by (job)"); - let QueryExpr::Aggregate { - reduction, child, .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - let QueryExpr::Aggregate { - reduction: inner_reduction, - .. - } = child.as_ref() - else { - panic!("expected inner Aggregate, got {child:?}"); - }; - // Inner output schema is [group, job, sum] (keys in label-column order, - // labels alphabetical on the scan) → job = col 1. - assert_eq!( - reduction, - &Reduction::by(vec![1]), - "outer `job` resolves against the inner output" - ); - assert_eq!(inner_reduction.expect_reduce().len(), 2); -} - -#[test] -fn outer_group_key_over_binary_op_resolves_on_both_sides() { - // Issue #52: an outer aggregate's group key that appears in *neither* side of - // a binary op — the metric-name label `__name__`, or a plain `job` — must - // still resolve. Each `or` side is bound independently against its own - // sub-DAG, so the key is seeded as an inherited column on both sides. - let qe = ok(r#"sum by (__name__)(metric_a{env="1"} or metric_b{env="2"})"#); - let QueryExpr::Aggregate { - reduction, child, .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - // `__name__` resolves to a single positional id against the binary op output. - assert_eq!( - reduction.expect_reduce().len(), - 1, - "grouped by the one `__name__` key" - ); - let QueryExpr::BinaryOp { lhs, rhs, .. } = child.as_ref() else { - panic!("expected a BinaryOp child, got {child:?}"); - }; - // Both independently-bound sides carry `__name__` at the same position, so - // the outer group key is consistent across the union. - let (ls, rs) = (lhs.output_schema().unwrap(), rhs.output_schema().unwrap()); - assert_eq!(ls.column_id("__name__"), rs.column_id("__name__")); - assert_eq!( - ls.column_id("__name__"), - Some(reduction.expect_reduce().keys()[0]) - ); - - // The general case (a plain label, not just `__name__`) also lowers. - assert!(matches!( - ok("sum by (job)(metric_a or metric_b)"), - QueryExpr::Aggregate { .. } - )); -} - -#[test] -fn aggregate_over_binary_op_nests() { - // `sum(rate(a[5m]) + rate(b[5m]))` — an aggregate whose argument is a binary - // op over two range vectors. The old template only accepted a single inner - // selector/call; now the binary op lowers and the outer sum wraps it. - let qe = ok("sum(rate(a[5m]) + rate(b[5m]))"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!( - matches!(child.as_ref(), QueryExpr::BinaryOp { .. }), - "argument lowers as a BinaryOp, got {child:?}" - ); -} - -// ───────────────────────────────────────────────────────────────────────────── -// J. Subqueries (basics §Subqueries; subquery.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn subquery_wraps_inner_query() { - // SEMANTICS: `[range:res]` evaluates the inner query across a range. - let qe = ok("rate(demo_api_request_duration_seconds_count[5m])[1h:]"); - assert!(matches!(&qe, QueryExpr::PromqlSubquery { .. })); - assert!(has(&qe, |i| matches!(i, AggIntent::Rate))); -} - -#[test] -fn over_time_of_subquery_reduces_per_series() { - // SEMANTICS (PromQL): `max_over_time(rate(...)[1h:])` chains a sub-query into - // a range-vector function — the sub-query evaluates `rate` across a 1h range, - // then `max_over_time` takes the max of those samples *per series*. It lowers - // to a per-series `Max` reduction over a `PromqlSubquery` (issue #27). - let qe = ok("max_over_time(rate(demo_api_request_duration_seconds_count[5m])[1h:])"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected an Aggregate at the root, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::PerEntity, - "`*_over_time` has no grouping — reduces per series" - ); - assert!(matches!(measures.as_slice(), [AggIntent::Max { .. }])); - // The reduction rides directly on the sub-query (the structural range marker - // that keeps it label-preserving), which wraps the inner `rate`. - assert!( - matches!(child.as_ref(), QueryExpr::PromqlSubquery { .. }), - "the `Max` reduces over a PromqlSubquery, got {child:?}" - ); - assert!(intents(&qe).iter().any(|i| matches!(i, AggIntent::Rate))); -} - -#[test] -fn quantile_over_time_of_subquery_carries_phi() { - // The `quantile_over_time` φ parameter is read from arg 0; the sub-query is - // arg 1. It lowers to a per-series `Quantile(φ)` over the `PromqlSubquery`. - let qe = ok("quantile_over_time(0.9, rate(demo[5m])[1h:])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected an Aggregate, got {qe:?}"); - }; - assert!( - matches!(measures.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.9).abs() < 1e-9) - ); - assert!(matches!(child.as_ref(), QueryExpr::PromqlSubquery { .. })); -} - -#[test] -fn aggregation_over_over_time_of_subquery_keeps_labels() { - // `sum by (job) (max_over_time(rate(m[5m])[1h:]))` — the inner - // `max_over_time` is per-series (label-preserving), so the `job` label - // survives for the OUTER cross-series `sum by (job)` to group on. If the - // inner `Max` collapsed labels, `job` would not resolve here. - let qe = ok("sum by (job) (max_over_time(rate(demo{job=\"api\"}[5m])[1h:]))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - assert!( - matches!(reduction, Reduction::Reduce(by) if !by.is_empty()), - "outer `sum by (job)` groups on a label" - ); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - // Inner node is the per-series `max_over_time` reduction over the subquery. - let QueryExpr::Aggregate { - reduction: inner_reduction, - measures: inner_measures, - child: inner_child, - .. - } = child.as_ref() - else { - panic!("expected inner Aggregate, got {child:?}"); - }; - assert_eq!(inner_reduction, &Reduction::PerEntity); - assert!(matches!(inner_measures.as_slice(), [AggIntent::Max { .. }])); - assert!(matches!( - inner_child.as_ref(), - QueryExpr::PromqlSubquery { .. } - )); -} - -#[test] -fn nested_subquery_from_prometheus_docs() { - // SEMANTICS (PromQL): the *nested sub-query* example from the official docs - // (): - // - // max_over_time(deriv(rate(distance_covered_total[5s])[30s:5s])[10m:]) - // - // Two stacked sub-queries, each feeding a range-vector function; the outer - // `[10m:]` uses the **default resolution** (no explicit step). Each level - // lowers to its own node, so the whole spine pins as: - // - // Max ∘ PromqlSubquery{10m, res: None} ∘ Deriv ∘ PromqlSubquery{30s, res: 5s} - // ∘ Rate ∘ TimeRange{5s} ∘ Scan - // - // Every reduction is per-series (no grouping), so the output schema stays - // the label-preserving `[ts, value]`. - let qe = ok("max_over_time(deriv(rate(distance_covered_total[5s])[30s:5s])[10m:])"); - - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected `max_over_time` Aggregate at the root, got {qe:?}"); - }; - assert_eq!(reduction, &Reduction::PerEntity); - assert!(matches!(measures.as_slice(), [AggIntent::Max { .. }])); - - let QueryExpr::PromqlSubquery { - range, - resolution, - child, - } = child.as_ref() - else { - panic!("expected the outer `[10m:]` PromqlSubquery, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(600)); - assert_eq!(*resolution, None, "`[10m:]` keeps the default resolution"); - - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = child.as_ref() - else { - panic!("expected the `deriv` Aggregate, got {child:?}"); - }; - assert_eq!(reduction, &Reduction::PerEntity); - assert!(matches!(measures.as_slice(), [AggIntent::Deriv])); - - let QueryExpr::PromqlSubquery { - range, - resolution, - child, - } = child.as_ref() - else { - panic!("expected the inner `[30s:5s]` PromqlSubquery, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(30)); - assert_eq!(*resolution, Some(Duration::from_secs(5))); - - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected the `rate` Aggregate, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Rate])); - let QueryExpr::TimeRange { range, .. } = child.as_ref() else { - panic!("expected the `[5s]` TimeRange under rate, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(5)); - - // Per-series end to end: the schema keeps the (ts, value) floor and stays open. - let schema = qe.output_schema().expect("schema derivation"); - assert_eq!( - schema - .fields - .iter() - .map(|c| c.name.as_str()) - .collect::>(), - vec!["ts", "value"], - ); - assert!(!schema.closed, "per-series chain never freezes the schema"); -} - -// ───────────────────────────────────────────────────────────────────────────── -// K. Time-shift modifiers (basics §Offset/@; at_modifier.test) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn offset_modifier_lowers_to_a_time_shift() { - // SEMANTICS (PromQL, issue #40): `offset 5m` shifts the lookback 5m into the - // past — a `TimeShift` wrapper over the selector (signed ms; a negative - // offset shifts forward). Schema is unchanged (the shift only moves *when*). - let qe = ok("http_requests_total offset 5m"); - let QueryExpr::TimeRange { child, .. } = &qe else { - panic!("expected an ingestion TimeRange, got {qe:?}"); - }; - let QueryExpr::TimeShift { shift, child } = child.as_ref() else { - panic!("expected a TimeShift, got {qe:?}"); - }; - assert_eq!(shift.offset_ms, 300_000); - assert!(shift.at.is_none()); - assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); - - // `offset -5m` shifts forward → negative ms. - let QueryExpr::TimeRange { child, .. } = &ok("http_requests_total offset -5m") else { - panic!("expected an ingestion TimeRange"); - }; - let QueryExpr::TimeShift { shift, .. } = child.as_ref() else { - panic!("expected a TimeShift"); - }; - assert_eq!(shift.offset_ms, -300_000); -} - -#[test] -fn at_modifier_lowers_to_a_time_shift() { - // SEMANTICS (PromQL, issue #40): `@ ` pins the evaluation to an absolute - // instant (PromQL seconds → IR milliseconds); `@ start()` / `@ end()` anchor - // to the query range bounds. - let QueryExpr::TimeRange { child, .. } = &ok("http_requests_total @ 1609746000") else { - panic!("expected an ingestion TimeRange"); - }; - let QueryExpr::TimeShift { shift, .. } = child.as_ref() else { - panic!("expected a TimeShift for `@ `"); - }; - assert_eq!(shift.at, Some(AtModifier::Timestamp(1_609_746_000_000))); - assert_eq!(shift.offset_ms, 0); - - let QueryExpr::TimeRange { child, .. } = &ok("http_requests_total @ start()") else { - panic!("expected an ingestion TimeRange"); - }; - let QueryExpr::TimeShift { shift, .. } = child.as_ref() else { - panic!("expected a TimeShift for `@ start()`"); - }; - assert_eq!(shift.at, Some(AtModifier::Start)); - - // Offset and `@` compose: `@ end() offset 5m` carries both. - let qe = ok("http_requests_total @ end() offset 5m"); - let QueryExpr::TimeRange { child, .. } = &qe else { - panic!("expected an ingestion TimeRange, got {qe:?}"); - }; - let QueryExpr::TimeShift { shift, .. } = child.as_ref() else { - panic!("expected a TimeShift, got {qe:?}"); - }; - assert_eq!(shift.at, Some(AtModifier::End)); - assert_eq!(shift.offset_ms, 300_000); -} - -#[test] -fn offset_on_a_ranged_selector_wraps_inside_the_time_range() { - // `rate(m[5m] offset 1h)` — the offset is on the ranged selector, so the - // `TimeShift` sits *under* the `TimeRange` (the 5m window is taken at the - // shifted time), and the whole thing under the per-series `Rate` (#40). - let qe = ok("rate(http_requests_total[5m] offset 1h)"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected the rate Aggregate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Rate])); - let QueryExpr::TimeRange { child, .. } = child.as_ref() else { - panic!("expected a TimeRange under rate, got {child:?}"); - }; - let QueryExpr::TimeShift { shift, child } = child.as_ref() else { - panic!("expected a TimeShift under the TimeRange, got {child:?}"); - }; - assert_eq!(shift.offset_ms, 3_600_000); - assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); -} - -// ───────────────────────────────────────────────────────────────────────────── -// L. Unsupported functions (functions.test) — clean rejection -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn unsupported_functions_are_rejected() { - // These parse fine but have no intent-algebra lowering yet. Each must return - // a clean LoweringError rather than mislower. - for q in [ - "step()", - "range()", - r#"histogram_quantiles("le", 0.5, 0.9, x)"#, - // NOTE: counter-derivatives (#44), math/trig (#45, §O), presence (#47, - // §P), time/calendar (#46, §Q), vector/scalar (#48, §R), - // label_replace/label_join (#50, §T) and the extra range reducers + - // sort family (#51, §U) now lower — see those sections. `info` (#84), - // `min_of`/`max_of` (#89) are pinned in §R / §U. - ] { - let _ = rejected(q); - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// M. Counter-derivative range functions (functions.test; issue #44) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn count_over_time_value_column_is_float64() { - // #69: a per-series range reduction produces a PromQL sample value, which is - // always float64. `count_over_time`'s `Count` intent types `Int64`, but the - // derived `value` column must be `Float64` like every other range reducer. - let schema = ok("count_over_time(m[5m])").output_schema().unwrap(); - let value = schema - .fields - .iter() - .find(|c| c.name == "value") - .expect("value column"); - assert_eq!(value.dtype, DataType::Float64); -} - -#[test] -fn counter_derivative_functions_lower_to_distinct_intents() { - // Each range function reduces one series' window to one value per series - // (label-preserving), riding on a `TimeRange`, and carries its OWN intent — - // deliberately not aliased to rate/increase/count. - for (q, want) in [ - ("changes(m[15m])", AggIntent::Changes), - ("delta(m[5m])", AggIntent::Delta), - ("idelta(m[5m])", AggIntent::IDelta), - ("deriv(m[1h])", AggIntent::Deriv), - ("resets(m[1h])", AggIntent::Resets), - ] { - let qe = ok(q); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected an Aggregate for {q:?}, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::PerEntity, - "{q}: per-series, no grouping" - ); - assert_eq!( - measures.as_slice(), - std::slice::from_ref(&want), - "{q}: wrong intent" - ); - assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { .. }), - "{q}: reduction rides on a TimeRange, got {child:?}" - ); - } -} - -#[test] -fn predict_linear_carries_horizon_seconds() { - // `predict_linear(v[w], t)` — the 2nd (scalar) arg is the prediction horizon - // in seconds; it must be carried in the intent (it changes the result). - let qe = ok("predict_linear(node_filesystem_avail_bytes[3h], 86400)"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected an Aggregate, got {qe:?}"); - }; - assert_eq!( - measures.as_slice(), - &[AggIntent::PredictLinear { seconds: 86400.0 }] - ); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -#[test] -fn double_exponential_smoothing_carries_factors() { - let want = AggIntent::DoubleExpSmoothing { - smoothing: 0.5, - trend: 0.3, - }; - let a = ok("double_exponential_smoothing(m[10m], 0.5, 0.3)"); - assert_eq!(intents(&a).as_slice(), std::slice::from_ref(&want)); -} - -#[test] -fn aggregation_over_counter_derivative_keeps_labels() { - // A counter-derivative is per-series (label-preserving), so an outer - // `sum by (job)` can group on a label the inner `changes` preserved. - let qe = ok(r#"sum by (job) (changes(m{job="api"}[15m]))"#); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - assert!( - matches!(reduction, Reduction::Reduce(by) if !by.is_empty()), - "outer sum groups on job" - ); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!(intents(&qe).iter().any(|i| matches!(i, AggIntent::Changes))); - let _ = child; -} - -#[test] -fn outer_stat_over_counter_derivative_nests_two_levels() { - // A cross-series stat over a counter-derivative is a genuine two-level - // reduction: the derivative runs per series (inner), the stat aggregates - // across series (outer). They must not collapse into one node — and a - // grouped outer (`avg by (dc)`) must resolve its key against the labels the - // inner reduction preserved, threading any scalar param (predict horizon). - let qe = ok("avg by (dc) (predict_linear(m[3h], 3600))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - assert!( - matches!(reduction, Reduction::Reduce(by) if !by.is_empty()), - "outer `avg by (dc)` groups on a label" - ); - assert!(matches!(measures.as_slice(), [AggIntent::Avg { .. }])); - let QueryExpr::Aggregate { - reduction: inner_reduction, - measures: inner_measures, - .. - } = child.as_ref() - else { - panic!("expected inner per-series Aggregate, got {child:?}"); - }; - assert_eq!( - inner_reduction, - &Reduction::PerEntity, - "inner derivative stays per-series" - ); - assert_eq!( - inner_measures.as_slice(), - std::slice::from_ref(&AggIntent::PredictLinear { seconds: 3600.0 }) - ); -} - -#[test] -fn topk_over_counter_derivative_is_generic_sort_limit() { - // `topk(k, deriv(...))` ranks the per-series derivative values — a generic - // `Sort + Limit`, NOT a heavy-hitter `TopK` (that's only `count_over_time`). - let qe = ok("topk(3, deriv(m[5m]))"); - let QueryExpr::Limit { n, child, .. } = &qe else { - panic!("expected Limit, got {qe:?}"); - }; - assert_eq!(*n, 3); - assert!(matches!(child.as_ref(), QueryExpr::Sort { .. })); - assert!(intents(&qe).iter().any(|i| matches!(i, AggIntent::Deriv))); - assert!( - !intents(&qe) - .iter() - .any(|i| matches!(i, AggIntent::TopK { .. })), - "counter-derivative topk is generic ranking, not a heavy-hitter sketch" - ); -} - -#[test] -fn counter_derivative_composes_in_binary_ops() { - // As a vector operand: `delta(a[5m]) / delta(b[5m])` is a BinaryOp of two - // per-series Delta reductions. - let ratio = ok("delta(a[5m]) / delta(b[5m])"); - let QueryExpr::BinaryOp { op, lhs, rhs, .. } = &ratio else { - panic!("expected BinaryOp, got {ratio:?}"); - }; - assert_eq!(*op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)); - assert!( - matches!(lhs.as_ref(), QueryExpr::Aggregate { measures, .. } if measures.as_slice() == [AggIntent::Delta]) - ); - assert!( - matches!(rhs.as_ref(), QueryExpr::Aggregate { measures, .. } if measures.as_slice() == [AggIntent::Delta]) - ); - - // Under an aggregate over a binary op mixing a counter-derivative with - // another per-series function: `sum(rate(m[5m]) + changes(m[5m]))`. - let mixed = ok("sum(rate(m[5m]) + changes(m[5m]))"); - let QueryExpr::Aggregate { - measures, child, .. - } = &mixed - else { - panic!("expected Aggregate, got {mixed:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::BinaryOp { .. })); - assert!(intents(&mixed).iter().any(|i| matches!(i, AggIntent::Rate))); - assert!(intents(&mixed) - .iter() - .any(|i| matches!(i, AggIntent::Changes))); -} - -#[test] -fn range_functions_over_a_subquery_reduce_per_series() { - // Issue #55 — the whole range-vector family accepts a sub-query argument - // (generalizing `*_over_time`, #42): `rate`/`increase`/`irate` and the - // counter-derivatives. Each lowers to a per-series `Aggregate{[f]}` directly - // over the `PromqlSubquery` — the sub-query is the range context, so there is NO - // separate `TimeRange` (that would double the range). - for (q, want) in [ - ("rate(sum(m)[5m:])", AggIntent::Rate), - ("increase(sum(m)[5m:])", AggIntent::Increase), - ("irate(sum(m)[5m:])", AggIntent::IRate), - ("changes(rate(m[5m])[1h:])", AggIntent::Changes), - ("delta(sum(m)[5m:])", AggIntent::Delta), - ("deriv(sum(m)[10m:])", AggIntent::Deriv), - ("resets(sum(m)[5m:])", AggIntent::Resets), - ] { - let qe = ok(q); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("{q}: expected an Aggregate, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::PerEntity, - "{q}: per-series, no grouping" - ); - assert_eq!( - measures.as_slice(), - std::slice::from_ref(&want), - "{q}: wrong intent" - ); - assert!( - matches!(child.as_ref(), QueryExpr::PromqlSubquery { .. }), - "{q}: reduces directly over the PromqlSubquery (no TimeRange), got {child:?}" - ); - } -} - -#[test] -fn predict_linear_and_double_exp_over_a_subquery_carry_params() { - // The scalar params survive the sub-query path. - let pl = ok("predict_linear(sum(m)[1h:], 3600)"); - assert!(intents(&pl).iter().any( - |i| matches!(i, AggIntent::PredictLinear { seconds } if (*seconds - 3600.0).abs() < 1e-9) - )); - let de = ok("double_exponential_smoothing(sum(m)[10m:], 0.5, 0.3)"); - assert!(intents(&de).iter().any(|i| matches!( - i, - AggIntent::DoubleExpSmoothing { smoothing, trend } - if (*smoothing - 0.5).abs() < 1e-9 && (*trend - 0.3).abs() < 1e-9 - ))); -} - -// ───────────────────────────────────────────────────────────────────────────── -// N. Native-histogram accessors (functions.test; issue #43) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn histogram_quantile_classic_bucket_vs_native() { - // Two lowerings of `histogram_quantile(φ, …)`: the classic cumulative-bucket - // form → exact `HistogramQuantile`; a native-histogram / raw-samples argument - // → the generic (sketch-able) `Quantile`. The classic form is recognised by - // `by (le)`, a `_bucket` metric, or an `le` matcher (issue #43). - for classic in [ - "histogram_quantile(0.9, sum by (le) (rate(x_bucket[5m])))", - "histogram_quantile(0.9, rate(x_bucket[5m]))", // bare _bucket metric - r#"histogram_quantile(0.9, rate(x{le="0.5"}[5m]))"#, // le matcher - ] { - let qe = ok(classic); - assert!( - has( - &qe, - |i| matches!(i, AggIntent::HistogramQuantile { q, .. } if (*q - 0.9).abs() < 1e-9) - ), - "classic bucket form → HistogramQuantile: {classic}" - ); - assert!( - !has(&qe, |i| matches!(i, AggIntent::Quantile { .. })), - "{classic}" - ); - } - for native in [ - "histogram_quantile(0.9, my_native_histogram)", - "histogram_quantile(0.9, request_duration_seconds)", // raw samples (your extension) - ] { - let qe = ok(native); - assert!( - has( - &qe, - |i| matches!(i, AggIntent::Quantile { q, .. } if (*q - 0.9).abs() < 1e-9) - ), - "native/raw form → generic Quantile: {native}" - ); - assert!( - !has(&qe, |i| matches!(i, AggIntent::HistogramQuantile { .. })), - "{native}" - ); - } -} - -#[test] -fn histogram_accessors_lower_to_per_series_intents() { - // `histogram_(v)` extracts a float per series from a native - // histogram — a per-series `Aggregate{[accessor]}` directly over the - // (instant) argument, no grouping. (`histogram_quantile` has its own two - // lowerings — see `histogram_quantile_classic_bucket_vs_native`.) - for (q, want) in [ - ("histogram_count(v)", AggIntent::HistogramCount), - ("histogram_sum(v)", AggIntent::HistogramSum), - ("histogram_avg(v)", AggIntent::HistogramAvg), - ("histogram_stddev(v)", AggIntent::HistogramStdDev), - ("histogram_stdvar(v)", AggIntent::HistogramStdVar), - ] { - let qe = ok(q); - let QueryExpr::Aggregate { - reduction, - measures, - .. - } = &qe - else { - panic!("{q}: expected an Aggregate, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::PerEntity, - "{q}: per-series, no grouping" - ); - assert_eq!( - measures.as_slice(), - std::slice::from_ref(&want), - "{q}: wrong intent" - ); - } -} - -#[test] -fn histogram_fraction_carries_its_bounds() { - // `histogram_fraction(lower, upper, v)` — bounds from args 0/1, vector arg 2. - let qe = ok("histogram_fraction(0, 0.2, v)"); - assert!(intents(&qe).iter().any(|i| matches!( - i, - AggIntent::HistogramFraction { lower, upper } - if *lower == 0.0 && (*upper - 0.2).abs() < 1e-9 - ))); -} - -// ───────────────────────────────────────────────────────────────────────────── -// O. Math / trig scalar-transform functions (functions.test; issue #45) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn math_functions_lower_to_per_series_math_intents() { - // Each `f(v)` is a per-series element-wise value transform — a per-series - // `Aggregate{[Math(f)]}` over the (instant) argument, no grouping. - for (q, want) in [ - ("abs(v)", MathFunc::Abs), - ("ceil(v)", MathFunc::Ceil), - ("floor(v)", MathFunc::Floor), - ("sqrt(v)", MathFunc::Sqrt), - ("ln(v)", MathFunc::Ln), - ("log2(v)", MathFunc::Log2), - ("sgn(v)", MathFunc::Sgn), - ("sin(v)", MathFunc::Sin), - ("atanh(v)", MathFunc::Atanh), - ("deg(v)", MathFunc::Deg), - ("rad(v)", MathFunc::Rad), - ] { - let qe = ok(q); - let QueryExpr::Aggregate { - reduction, - measures, - .. - } = &qe - else { - panic!("{q}: expected an Aggregate, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::PerEntity, - "{q}: per-series, no grouping" - ); - assert!( - matches!(measures.as_slice(), [AggIntent::Math(m)] if *m == want), - "{q}: wrong intent, got {measures:?}" - ); - } -} - -#[test] -fn clamp_and_round_carry_their_params() { - assert!(intents(&ok("clamp(v, 0, 100)")).iter().any( - |i| matches!(i, AggIntent::Math(MathFunc::Clamp { min, max }) if *min == 0.0 && *max == 100.0) - )); - assert!(intents(&ok("clamp_min(v, 1)")) - .iter() - .any(|i| matches!(i, AggIntent::Math(MathFunc::ClampMin { min }) if *min == 1.0))); - assert!(intents(&ok("clamp_max(v, 5)")) - .iter() - .any(|i| matches!(i, AggIntent::Math(MathFunc::ClampMax { max }) if *max == 5.0))); - // `round(v)` defaults the step to 1; `round(v, 5)` reads it. - assert!(intents(&ok("round(v)")).iter().any( - |i| matches!(i, AggIntent::Math(MathFunc::Round { to_nearest }) if *to_nearest == 1.0) - )); - assert!(intents(&ok("round(v, 5)")).iter().any( - |i| matches!(i, AggIntent::Math(MathFunc::Round { to_nearest }) if *to_nearest == 5.0) - )); -} - -#[test] -fn pi_lowers_to_a_scalar_constant() { - // `pi()` is the constant π — a `PromqlScalarBridge` leaf, not a `Math` intent. - assert!(ok("pi()") - .as_promql_scalar() - .is_some_and(|v| (v - std::f64::consts::PI).abs() < 1e-12)); -} - -// ───────────────────────────────────────────────────────────────────────────── -// P. Presence functions (functions.test; issue #47) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn presence_functions_lower_to_presence_intents() { - for (q, want) in [ - (r#"absent(up{job="x"})"#, AggIntent::Absent), - ("absent_over_time(m[1h])", AggIntent::AbsentOverTime), - ("present_over_time(m[5m])", AggIntent::PresentOverTime), - ] { - let qe = ok(q); - assert!(intents(&qe).contains(&want), "{q}: got {:?}", intents(&qe)); - } -} - -#[test] -fn absent_keeps_matcher_labels_for_the_synthesized_output() { - // `absent(v)` synthesizes its output labels from `v`'s equality matchers, so - // those labels must survive into the schema — here `job` from `{job="x"}`. - let qe = ok(r#"absent(up{job="x"})"#); - let cols = qe.output_schema().unwrap(); - assert!( - cols.fields.iter().any(|c| c.name == "job"), - "matcher label `job` kept, got {:?}", - cols.fields.iter().map(|c| &c.name).collect::>() - ); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Q. Time / calendar functions (functions.test; issue #46) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn time_lowers_to_the_eval_time_scalar() { - // SEMANTICS: `time()` is the query evaluation timestamp as a scalar — a leaf, - // not an aggregate over any series. - assert!(matches!(ok("time()"), QueryExpr::EvalTimestamp)); - // …and it is scalar-shaped: a single float `value`, no time index. - let sch = ok("time()").output_schema().unwrap(); - assert_eq!(sch.fields.len(), 1); - assert_eq!(sch.fields[0].name, "value"); - assert!(sch.time_index.is_none()); -} - -#[test] -fn time_minus_vector_is_the_uptime_pattern() { - // `time() - process_start_time_seconds` — the canonical uptime expression. - // The scalar `time()` broadcasts against the vector; the result takes the - // vector's schema. - let qe = ok("time() - process_start_time_seconds"); - let QueryExpr::BinaryOp { lhs, op, .. } = &qe else { - panic!("expected a BinaryOp, got {qe:?}"); - }; - assert!(matches!(lhs.as_ref(), QueryExpr::EvalTimestamp)); - assert!(matches!( - op, - BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub) - )); - assert!(qe.output_schema().is_ok()); -} - -#[test] -fn calendar_functions_lower_to_time_fn_intents() { - // SEMANTICS: each of these is a per-series float transform of its argument's - // timestamp (or, for `timestamp`, the sample's own time). functions.test. - for (q, want) in [ - ("timestamp(up)", TimeFunc::Timestamp), - ("minute(v)", TimeFunc::Minute), - ("hour(v)", TimeFunc::Hour), - ("day_of_week(v)", TimeFunc::DayOfWeek), - ("day_of_month(v)", TimeFunc::DayOfMonth), - ("day_of_year(v)", TimeFunc::DayOfYear), - ("month(v)", TimeFunc::Month), - ("year(v)", TimeFunc::Year), - ("days_in_month(v)", TimeFunc::DaysInMonth), - ] { - let qe = ok(q); - assert!( - has(&qe, |i| *i == AggIntent::TimeFn(want)), - "{q} → TimeFn({want:?}), got {:?}", - intents(&qe) - ); - } -} - -#[test] -fn no_arg_calendar_function_reads_the_eval_time() { - // `day_of_week()` with no argument computes over the evaluation time itself, - // so it is a `TimeFn` aggregate whose child is the `EvalTimestamp` scalar. - let qe = ok("day_of_week()"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected an Aggregate, got {qe:?}"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::TimeFn(TimeFunc::DayOfWeek)] - )); - assert!(matches!(child.as_ref(), QueryExpr::EvalTimestamp)); -} - -#[test] -fn timestamp_composes_under_an_outer_aggregation() { - // `sum by (job) (timestamp(up))` — the per-series `timestamp` transform sits - // below an ordinary grouped sum. Both intents must appear in the DAG. - let qe = ok("sum by (job) (timestamp(up))"); - assert!(has(&qe, |i| *i == AggIntent::TimeFn(TimeFunc::Timestamp))); - assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); -} - -// ───────────────────────────────────────────────────────────────────────────── -// R. Type-conversion functions: vector() / scalar() (functions.test; issue #48) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn vector_promotes_a_scalar_to_a_vector() { - // SEMANTICS: `vector(s)` is the scalar→instant-vector bridge — a label-less - // single series carrying the scalar's value. - let qe = ok("vector(1)"); - let QueryExpr::PromqlVectorFromScalar(inner) = &qe else { - panic!("expected PromqlVectorFromScalar, got {qe:?}"); - }; - assert_eq!(inner.as_promql_scalar(), Some(1.0)); - // Vector-typed: schema has a time index (a scalar leaf has none). - let sch = qe.output_schema().unwrap(); - assert!(sch.time_index.is_some()); - assert!(sch.fields.iter().any(|c| c.name == "value")); -} - -#[test] -fn scalar_collapses_a_vector_to_a_scalar() { - // SEMANTICS: `scalar(v)` is the instant-vector→scalar bridge. - let qe = ok("scalar(node_load1)"); - let QueryExpr::PromqlScalarFromVector(inner) = &qe else { - panic!("expected PromqlScalarFromVector, got {qe:?}"); - }; - let (metric, _) = first_scan(inner); - assert_eq!(metric, "node_load1"); - // PromqlScalarBridge-typed: single `value` column, no time index. - let sch = qe.output_schema().unwrap(); - assert!(sch.time_index.is_none()); - assert_eq!(sch.fields.len(), 1); - assert_eq!(sch.fields[0].name, "value"); -} - -#[test] -fn vector_zero_is_a_vector_operand_of_a_set_op() { - // `up or vector(0)` — the dead-man's-switch. `or` is a set op between two - // vectors, so `vector(0)` must be a vector (a `PromqlVectorFromScalar`), never a - // folded scalar operand. - let qe = ok("up or vector(0)"); - let QueryExpr::BinaryOp { rhs, op, .. } = &qe else { - panic!("expected a BinaryOp, got {qe:?}"); - }; - assert_eq!(*op, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)); - assert!(matches!(rhs.as_ref(), QueryExpr::PromqlVectorFromScalar(_))); -} - -#[test] -fn scalar_of_a_vector_feeds_a_threshold_comparison() { - // `node_load1 > scalar(node_cpu_count)` — `scalar(...)` is a scalar operand, - // so the BinaryOp output takes the vector (lhs) side's schema. - let qe = ok("node_load1 > scalar(node_cpu_count)"); - let QueryExpr::BinaryOp { lhs, rhs, .. } = &qe else { - panic!("expected a BinaryOp, got {qe:?}"); - }; - assert!(matches!(rhs.as_ref(), QueryExpr::PromqlScalarFromVector(_))); - // The BinaryOp output schema follows the vector (lhs) side, not the scalar. - let (metric, _) = first_scan(lhs); - assert_eq!(metric, "node_load1"); - assert!(qe.output_schema().unwrap().time_index.is_some()); -} - -#[test] -fn info_lowers_to_a_label_enrichment_join() { - // `info(v, [selector])` is a label-enrichment *join* against the info - // metric(s) — it lowers to an `PromqlInfoEnrich` over the (unchanged) input vector - // (issue #84). The value/time axis pass through; the enriched labels are - // runtime, so the schema stays the child's. - let qe = ok("info(rate(http_requests_total[5m]))"); - let QueryExpr::PromqlInfoEnrich { selector, child } = &qe else { - panic!("expected an PromqlInfoEnrich, got {qe:?}"); - }; - assert!(selector.is_empty(), "no selector → default target_info"); - // The child is the untouched input (a per-series rate reduction here). - assert!(has(child, |i| *i == AggIntent::Rate)); - assert!(qe.output_schema().unwrap().time_index.is_some()); -} - -#[test] -fn info_selector_carries_the_info_side_matchers() { - // `info(v, {__name__=~".+_info", data=~".+"})` — the selector picks the info - // metric(s) via `__name__` and constrains the data labels. Regex / `__name__` - // matchers are kept symbolically (not run through the single-metric selector - // path). - let qe = ok(r#"info(build_info, {__name__=~".+_info", another_data=~".+"})"#); - let QueryExpr::PromqlInfoEnrich { selector, .. } = &qe else { - panic!("expected an PromqlInfoEnrich, got {qe:?}"); - }; - assert_eq!( - selector.len(), - 2, - "both selector matchers kept: {selector:?}" - ); - assert!(selector - .iter() - .any(|m| m.label == "__name__" && m.op == CompareOpKind::Regex)); - assert!(selector.iter().any(|m| m.label == "another_data")); -} - -#[test] -fn info_composes_under_an_aggregation_and_over_a_time_shift() { - // `sum(info(m))` — enrichment first, then a cross-series sum over it. - assert!(has(&ok("sum(info(node_uname_info))"), |i| matches!( - i, - AggIntent::Sum { .. } - ))); - // `offset` / `@` on the input now lower to a `TimeShift` under the info-join - // (issue #40) — the enrichment composes over the shifted selector. - assert!(matches!( - ok("info(metric @ 60)"), - QueryExpr::PromqlInfoEnrich { .. } - )); - assert!(matches!( - ok("info(metric offset 1m)"), - QueryExpr::PromqlInfoEnrich { .. } - )); -} - -// ───────────────────────────────────────────────────────────────────────────── -// S. Extended aggregation operators: group / count_values (aggregators.test; #49) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn group_lowers_to_a_constant_group_intent() { - // SEMANTICS: `group(v)` yields a constant 1 per group — a distinct intent, - // NOT folded onto `sum` (which would return the value sum instead of 1). - let qe = ok("group(up)"); - let QueryExpr::Aggregate { measures, .. } = &qe else { - panic!("expected an Aggregate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Group])); - // Output column is the constant-1 `group` value. - let sch = qe.output_schema().unwrap(); - assert!(sch.fields.iter().any(|c| c.name == "group")); -} - -#[test] -fn group_by_keeps_the_grouping_keys() { - // `group by (job) (up)` — the grouping keys ride on `Aggregate.by`. - let qe = ok("group by (job) (up)"); - let sch = qe.output_schema().unwrap(); - assert!(sch.fields.iter().any(|c| c.name == "job")); - assert!(has(&qe, |i| *i == AggIntent::Group)); -} - -#[test] -fn count_values_groups_by_value_and_synthesizes_a_label() { - // SEMANTICS: `count_values("l", v)` groups the input series by their sample - // value, counts each distinct value, and emits that value as a new label - // `l`. The intent carries the label; schema gains a `Utf8` `l` column. - let qe = ok(r#"count_values("version", build_version)"#); - let QueryExpr::Aggregate { measures, .. } = &qe else { - panic!("expected an Aggregate, got {qe:?}"); - }; - assert!( - matches!(measures.as_slice(), [AggIntent::CountValues { label }] if label == "version") - ); - let sch = qe.output_schema().unwrap(); - let version = sch - .fields - .iter() - .find(|c| c.name == "version") - .expect("synthesized `version` label column"); - assert_eq!( - version.dtype, - DataType::Utf8, - "the value becomes a string label" - ); - assert!( - sch.fields.iter().any(|c| c.name == "count"), - "and a count column" - ); -} - -#[test] -fn count_values_accepts_a_parenthesised_label_and_by_grouping() { - // `count_values by (job) ((("v")), m)` — nested parens around the string - // param, plus `by` grouping. Both survive. - let qe = ok(r#"count_values by (job) ((("v")), m)"#); - assert!(has( - &qe, - |i| matches!(i, AggIntent::CountValues { label } if label == "v") - )); - let sch = qe.output_schema().unwrap(); - assert!(sch.fields.iter().any(|c| c.name == "job")); - assert!(sch.fields.iter().any(|c| c.name == "v")); -} - -#[test] -fn count_values_label_colliding_with_a_group_key_is_not_duplicated() { - // `count_values by (job)("job", v)` — the synthesized label name collides - // with a group-by key. PromQL's synthesized label takes precedence; the - // output must carry a single `job` column, never two. - let qe = ok(r#"count_values by (job) ("job", version)"#); - let sch = qe.output_schema().unwrap(); - let jobs = sch.fields.iter().filter(|c| c.name == "job").count(); - assert_eq!(jobs, 1, "collision deduped, got {:?}", sch.fields); - assert!(sch.fields.iter().any(|c| c.name == "count")); -} - -#[test] -fn limitk_and_limit_ratio_lower_to_series_sampling() { - // `limitk`/`limit_ratio` are series-*sampling* selection — a subset of whole - // series kept unchanged (NOT a ranking), so they lower to the dedicated - // `PromqlSeriesSample` node, never `topk`'s `Sort → Limit` (issue #86). - assert!(matches!( - ok("limitk(2, http_requests)"), - QueryExpr::PromqlSeriesSample { - kind: SampleKind::LimitK(2), - .. - } - )); - assert!(matches!( - ok("limit_ratio(0.1, http_requests)"), - QueryExpr::PromqlSeriesSample { kind: SampleKind::LimitRatio(r), .. } if (r - 0.1).abs() < 1e-9 - )); - // Series-preserving: the output schema equals the input's (ts, value). - let sch = ok("limitk(2, http_requests)").output_schema().unwrap(); - assert!(sch.fields.iter().any(|c| c.name == "value")); - assert!(sch.time_index.is_some()); -} - -#[test] -fn limit_ratio_keeps_a_negative_ratio_and_clamps_out_of_range() { - // A negative ratio selects the complementary fraction — it must survive, not - // be normalised away. Out-of-range magnitudes clamp to [-1, 1] (Prometheus). - assert!(matches!( - ok("limit_ratio(-0.5, http_requests)"), - QueryExpr::PromqlSeriesSample { kind: SampleKind::LimitRatio(r), .. } if (r + 0.5).abs() < 1e-9 - )); - assert!(matches!( - ok("limit_ratio(1.1, http_requests)"), - QueryExpr::PromqlSeriesSample { kind: SampleKind::LimitRatio(r), .. } if (r - 1.0).abs() < 1e-9 - )); -} - -#[test] -fn limitk_by_carries_the_grouping_and_composes_in_a_set_op() { - // `limitk by (group)` samples per group; the grouping label is seeded. - let qe = ok("limitk by (group) (2, http_requests)"); - let QueryExpr::PromqlSeriesSample { by, .. } = &qe else { - panic!("expected a PromqlSeriesSample, got {qe:?}"); - }; - assert!(!by.is_empty(), "grouped sampling keeps its `by` keys"); - // `count(limitk(2, v) and v)` — the surviving series' identity matters, so - // the PromqlSeriesSample must be preserved under the set op (it must lower, not reject). - assert!(has( - &ok("count(limitk(2, http_requests) and http_requests)"), - |i| matches!(i, AggIntent::Count { .. }) - )); -} - -#[test] -fn dynamic_and_non_finite_sample_params_are_rejected() { - // A dynamic k/ratio (not a compile-time constant) or a NaN can't be a static - // `PromqlSeriesSample` param — rejected rather than mislowered. - let _ = rejected("limitk(NaN, http_requests)"); - let _ = rejected("limitk(scalar(foo), http_requests)"); - let _ = rejected("limit_ratio(time() % 17 / 17, http_requests)"); -} - -// ───────────────────────────────────────────────────────────────────────────── -// T. Label-rewrite functions: label_replace / label_join (functions.test; #50) -// ───────────────────────────────────────────────────────────────────────────── - -/// Descend single-child nodes to the first `PromqlRelabel`. -fn first_relabel(e: &QueryExpr) -> &QueryExpr { - match e { - QueryExpr::PromqlRelabel { .. } => e, - QueryExpr::Aggregate { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } => first_relabel(child), - other => panic!("no PromqlRelabel reachable from {other:?}"), - } -} - -/// True when `value` is a `FunctionCall` with the given name. -fn is_fn_named(value: &QueryExpr, name: &str) -> bool { - matches!(value, QueryExpr::FunctionCall { name: n, .. } if n == name) -} - -#[test] -fn label_replace_is_a_relabel_over_the_vector() { - // SEMANTICS: `label_replace(v, dst, repl, src, regex)` rewrites the `dst` - // label per series from a regex over `src`; the sample value is untouched. - let qe = ok(r#"label_replace(up, "host", "$1", "instance", "(.+):.*")"#); - let QueryExpr::PromqlRelabel { dst, value, child } = &qe else { - panic!("expected a PromqlRelabel, got {qe:?}"); - }; - assert_eq!(dst, "host"); - // The child is the untouched vector. - let (metric, _) = first_scan(child); - assert_eq!(metric, "up"); - // The value expression is a `label_replace` fn reading the `src` label. - assert!(is_fn_named(value, "label_replace")); - // Output: the child's columns + the synthesized `host` label; value & ts kept. - let sch = qe.output_schema().unwrap(); - assert!(sch.fields.iter().any(|c| c.name == "host")); - assert!(sch.fields.iter().any(|c| c.name == "value")); - assert!(sch.time_index.is_some(), "the vector's time axis survives"); -} - -#[test] -fn label_join_concatenates_source_labels() { - // SEMANTICS: `label_join(v, dst, sep, src…)` joins the source labels with - // `sep` into `dst`. - let qe = ok(r#"label_join(up, "combined", "-", "job", "instance")"#); - let QueryExpr::PromqlRelabel { dst, value, .. } = &qe else { - panic!("expected a PromqlRelabel, got {qe:?}"); - }; - assert_eq!(dst, "combined"); - assert!(is_fn_named(value, "label_join")); - let sch = qe.output_schema().unwrap(); - assert!(sch.fields.iter().any(|c| c.name == "combined")); -} - -#[test] -fn label_replace_composes_under_an_aggregation() { - // `sum by (host) (label_replace(up, "host", "$1", "instance", "(.+):.*"))` — - // relabel first, then group by the synthesized label. - let qe = ok(r#"sum by (host) (label_replace(up, "host", "$1", "instance", "(.+):.*"))"#); - // A PromqlRelabel sits below the outer Sum. - let relabel = first_relabel(&qe); - assert!(matches!(relabel, QueryExpr::PromqlRelabel { dst, .. } if dst == "host")); - assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); - let sch = qe.output_schema().unwrap(); - assert!(sch.fields.iter().any(|c| c.name == "host")); -} - -// ───────────────────────────────────────────────────────────────────────────── -// U. Long-tail: extra range reducers + the sort family (functions.test; #51) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn extra_over_time_reducers_lower_to_per_series_intents() { - // SEMANTICS: each is a per-series reduction of one series' range window to a - // single value — a `TimeRange`-wrapped `Aggregate` with the matching intent. - for (q, want) in [ - ("last_over_time(m[5m])", AggIntent::LastOverTime), - ("first_over_time(m[5m])", AggIntent::FirstOverTime), - ("mad_over_time(m[5m])", AggIntent::MadOverTime), - ("ts_of_min_over_time(m[5m])", AggIntent::TsOfMinOverTime), - ("ts_of_max_over_time(m[5m])", AggIntent::TsOfMaxOverTime), - ("ts_of_first_over_time(m[5m])", AggIntent::TsOfFirstOverTime), - ("ts_of_last_over_time(m[5m])", AggIntent::TsOfLastOverTime), - ] { - let qe = ok(q); - assert!(has(&qe, |i| *i == want), "{q}: {:?}", intents(&qe)); - // Per-series: the range window survives as a `TimeRange`. - assert!( - matches!(&qe, QueryExpr::Aggregate { child, .. } if matches!(child.as_ref(), QueryExpr::TimeRange { .. })), - "{q} keeps its range as a TimeRange" - ); - } -} - -#[test] -fn last_over_time_composes_under_an_outer_aggregation() { - // `sum by (job) (last_over_time(m[5m]))` — per-series last, THEN cross-series - // sum. Both intents survive (issue #27's arbitrary nesting). - let qe = ok("sum by (job) (last_over_time(m[5m]))"); - assert!(has(&qe, |i| *i == AggIntent::LastOverTime)); - assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); -} - -#[test] -fn sort_and_sort_desc_reorder_by_value_without_a_limit() { - // SEMANTICS: `sort`/`sort_desc` reorder an instant vector by sample value. - // Row-preserving → a bare `Sort` (no `Limit`), ascending / descending. - for (q, ascending) in [ - ("sort(http_requests)", true), - ("sort_desc(http_requests)", false), - ] { - let qe = ok(q); - let QueryExpr::Sort { keys, child, .. } = &qe else { - panic!("{q}: expected a Sort, got {qe:?}"); - }; - assert_eq!(keys.len(), 1); - assert_eq!(keys[0].ascending, ascending, "{q}"); - // No Limit above the Sort — every series is preserved. - assert!(!matches!(&qe, QueryExpr::Limit { .. })); - // The value column is what it ranks on: descend to the scan. - let (metric, _) = first_scan(child); - assert_eq!(metric, "http_requests"); - } -} - -#[test] -fn sort_by_label_orders_on_each_label_in_turn() { - // `sort_by_label(v, "group", "instance", "job")` — one ascending sort key per - // label, in argument order; the labels are seeded into the schema. - let qe = ok(r#"sort_by_label(http_requests, "group", "instance", "job")"#); - let QueryExpr::Sort { keys, .. } = &qe else { - panic!("expected a Sort, got {qe:?}"); - }; - assert_eq!(keys.len(), 3, "one key per label"); - assert!(keys.iter().all(|k| k.ascending)); - let sch = qe.output_schema().unwrap(); - for label in ["group", "instance", "job"] { - assert!(sch.fields.iter().any(|c| c.name == label), "{label} seeded"); - } -} - -#[test] -fn sort_by_label_desc_is_descending() { - let qe = ok(r#"sort_by_label_desc(http_requests, "instance")"#); - let QueryExpr::Sort { keys, .. } = &qe else { - panic!("expected a Sort, got {qe:?}"); - }; - assert!(keys.iter().all(|k| !k.ascending)); -} - -#[test] -fn min_of_max_of_fold_constant_scalars() { - // `min_of`/`max_of` are n-ary scalar reducers. When every argument is a - // constant they constant-fold to a `PromqlScalarBridge` leaf, just like scalar - // arithmetic (#35) — the only form the intent algebra can hold (#89). - assert_eq!(ok("min_of(3, 5)").as_promql_scalar(), Some(3.0)); - assert_eq!(ok("max_of(3, 5)").as_promql_scalar(), Some(5.0)); - assert_eq!(ok("min_of(-2, -5)").as_promql_scalar(), Some(-5.0)); - // Nested folds and use as a threshold operand. - assert_eq!( - ok("max_of(min_of(2, 3), 10)").as_promql_scalar(), - Some(10.0) - ); - let qe = ok("up > max_of(1, 2)"); - let QueryExpr::BinaryOp { rhs, .. } = &qe else { - panic!("{qe:?}") - }; - assert_eq!(rhs.as_promql_scalar(), Some(2.0)); -} - -#[test] -fn min_of_max_of_ignore_nan_like_the_min_max_aggregators() { - // A NaN argument is skipped (Prometheus `min`/`max` NaN semantics). - assert_eq!(ok("max_of(3, NaN)").as_promql_scalar(), Some(3.0)); - assert_eq!(ok("min_of(NaN, 3)").as_promql_scalar(), Some(3.0)); -} - -#[test] -fn non_constant_min_of_max_of_is_rejected__GAP() { - // A dynamic argument (`step()` — itself unsupported, #89) can't be folded to - // a constant and there is no scalar min/max node, so it stays rejected - // rather than mislowered. These forms also only appear inside unsupported - // dynamic range / offset positions in the corpus. - let _ = rejected("min_of(step(), 1s)"); - let _ = rejected("max_of(min_of(step() + 1, 1h), 1ms)"); -} diff --git a/crates/frontend-promql/tests/promql_lowering.rs b/crates/frontend-promql/tests/promql_lowering.rs deleted file mode 100644 index 243bf6d18..000000000 --- a/crates/frontend-promql/tests/promql_lowering.rs +++ /dev/null @@ -1,1346 +0,0 @@ -//! End-to-end tests for PromQL → unresolved → canonical DAG lowering. - -use std::time::Duration; - -use asap_types::pre_asap::{ - AggIntent, ArithmeticOpKind, BinaryOpKind, CompareOpKind, QueryExpr, Reduction, ScalarValue, - Source, -}; -use asap_types::types::AccuracyTarget; -use asap_types::workload::{ - AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, PlanningWorkload, - Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, TimeSelection, -}; - -use asap_frontend_promql::{lower_promql_workload, PromqlError as LoweringError}; -mod support; -use support::lower_promql; - -fn lower(q: &str) -> QueryExpr { - lower_promql(q, AccuracyTarget::Exact).unwrap_or_else(|e| panic!("lower failed for {q:?}: {e}")) -} - -#[test] -fn frequency_extensions_lower_to_explicit_frequency_statistics() { - // ProjectASAP extensions reduce a frequency vector; numeric sample norms - // have different semantics and must never be silently aliased here. - for (query, expected) in [ - ( - "entropy_over_time(cpu_usage[5m])", - AggIntent::FrequencyEntropy { - col: None, - accuracy: AccuracyTarget::Exact, - }, - ), - ( - "l2_over_time(cpu_usage[5m])", - AggIntent::FrequencyL2 { - col: None, - accuracy: AccuracyTarget::Exact, - }, - ), - ] { - assert!(all_intents(&lower(query)) - .iter() - .any(|intent| std::mem::discriminant(intent) == std::mem::discriminant(&expected))); - } -} - -#[test] -fn distinct_over_time_preserves_cardinality_accuracy_and_nested_windows() { - // Distinct counts sample values, not samples or series; all lowering routes - // retain the caller's accuracy requirement, including subquery arguments. - for query in [ - "distinct_over_time(cpu_usage{job=\"worker\"}[5m] offset 1h)", - "distinct_over_time((cpu_usage + 1)[5m:1m])", - "sum by(job)(distinct_over_time(cpu_usage[5m]))", - ] { - for accuracy in [AccuracyTarget::Exact, AccuracyTarget::Epsilon(0.02)] { - let dag = lower_promql(query, accuracy.clone()).unwrap(); - let mut intents = Vec::new(); - collect_intents(&dag, &mut intents); - assert!( - intents.iter().any(|intent| matches!( - intent, AggIntent::Cardinality { accuracy: actual, .. } if actual == &accuracy - )), - "{query}: {dag:?}" - ); - assert!(!intents - .iter() - .any(|intent| matches!(intent, AggIntent::Count { .. }))); - } - } -} - -// ── Bare selectors & label matchers (folded onto Scan.predicates) ─────────────── - -#[test] -fn bare_selector_is_scan_with_predicates() { - let qe = lower(r#"http_requests_total{env="prod",status!="500"}"#); - let QueryExpr::TimeRange { child, .. } = &qe else { - panic!("expected TimeRange, got {qe:?}"); - }; - let QueryExpr::Scan { - source, predicates, .. - } = child.as_ref() - else { - panic!("expected Scan, got {qe:?}"); - }; - assert!(matches!(source, Source::TimeSeries { metric } if metric == "http_requests_total")); - // The converter splits the matcher conjunction into one predicate per - // conjunct on the Scan. - assert_eq!(predicates.len(), 2); - assert!(predicates - .iter() - .all(|p| matches!(p.0.as_ref(), QueryExpr::Compare { .. }))); -} - -#[test] -fn regex_matcher_lowers_to_regex_compareop() { - let qe = lower(r#"http_requests_total{path=~"/api/.*"}"#); - let QueryExpr::TimeRange { child, .. } = &qe else { - panic!("expected TimeRange, got {qe:?}"); - }; - let QueryExpr::Scan { - predicates, schema, .. - } = child.as_ref() - else { - panic!("expected Scan, got {qe:?}"); - }; - let QueryExpr::Compare { left, op, right } = predicates[0].0.as_ref() else { - panic!("expected Compare, got {:?}", predicates[0].0); - }; - assert_eq!(*op, CompareOpKind::Regex); - // The label matcher's column is resolved positionally against the scan schema. - let path_id = schema.column_id("path").expect("path in scan schema"); - assert!(matches!(left.as_ref(), QueryExpr::Column(id) if *id == path_id)); - assert!(matches!(right.as_ref(), QueryExpr::Literal(ScalarValue::Utf8(v)) if v == "/api/.*")); -} - -// ── *_over_time → Aggregate over TimeRange ────────────────────────────────────── - -#[test] -fn quantile_over_time_is_time_range_aggregate() { - let qe = lower(r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected Aggregate, got {qe:?}"); - }; - assert_eq!(reduction, &Reduction::PerEntity); - assert!( - matches!(measures.as_slice(), [AggIntent::Quantile { q, .. }] if (*q - 0.99).abs() < 1e-9) - ); - let QueryExpr::TimeRange { range, child } = child.as_ref() else { - panic!("expected TimeRange child, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(300)); - // The label matcher folded onto the Scan. - assert!(matches!(child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1)); -} - -#[test] -fn outer_sum_by_over_quantile_over_time_groups_positionally() { - // `sum by (host) (quantile_over_time(...))`: inner per-series - // quantile-over-time (label-preserving), then an outer cross-series sum - // grouped on a positional `Aggregate.by` — the same shape SQL produces, not - // a name-based Partition. Leaf = [ts, value, host, service] (referenced - // names appended sorted) → host = col 2. - let qe = lower(r#"sum by (host) (quantile_over_time(0.99, latency{service="web"}[5m]))"#); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate grouped by host, got {qe:?}"); - }; - assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - // Inner: Aggregate{Quantile} over TimeRange (per-series over_time reduction). - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected Aggregate (quantile_over_time) under the outer Sum, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Quantile { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -#[test] -fn avg_over_time_maps_to_avg_intent() { - let qe = lower("avg_over_time(cpu_seconds_total[10m])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Avg { .. }])); - let QueryExpr::TimeRange { range, .. } = child.as_ref() else { - panic!("expected TimeRange child, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(600)); -} - -#[test] -fn stddev_and_stdvar_over_time() { - let qe = lower("stddev_over_time(m[5m])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::StdDev { - population: true, - .. - }] - )); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); - - let qe = lower("stdvar_over_time(m[5m])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::Variance { - population: true, - .. - }] - )); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -#[test] -fn histogram_quantile_wraps_inner_in_quantile() { - // The argument's structure (here `rate`) is preserved *under* the quantile, - // not squashed away. The `_bucket` metric + `le` matcher mark the classic - // form → `HistogramQuantile` over `Aggregate{Rate}` over Scan. - let qe = lower(r#"histogram_quantile(0.95, rate(http_duration_seconds_bucket{le="0.5"}[5m]))"#); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate{{HistogramQuantile}}, got {qe:?}"); - }; - assert!( - matches!(measures.as_slice(), [AggIntent::HistogramQuantile { q, .. }] if (*q - 0.95).abs() < 1e-9) - ); - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected inner Aggregate{{Rate}}, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Rate])); - let QueryExpr::TimeRange { - range, - child: tr_child, - } = child.as_ref() - else { - panic!("expected TimeRange under Rate, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(300)); - assert!( - matches!(tr_child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1) - ); -} - -#[test] -fn histogram_quantile_over_sum_by_le_preserves_grouping() { - // The canonical Prometheus histogram pattern. Previously returned - // UnsupportedFeature because `extract_matrix` couldn't see through the - // `sum by (le)` aggregate; now the `le` grouping survives into the - // canonical DAG. - let qe = lower(r#"histogram_quantile(0.99, sum by (le) (rate(http_requests_bucket[5m])))"#); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate{{HistogramQuantile}}, got {qe:?}"); - }; - // The `by (le)` grouping marks the classic cumulative-bucket form. - assert!( - matches!(measures.as_slice(), [AggIntent::HistogramQuantile { q, .. }] if (*q - 0.99).abs() < 1e-9) - ); - // `sum by (le)` survives as a positional Aggregate (by = [2], `le`) over the - // inner Rate — no name-based Partition. - let QueryExpr::Aggregate { - reduction, - measures, - .. - } = child.as_ref() - else { - panic!("expected `sum by (le)` as a positional Aggregate, got {child:?}"); - }; - assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); -} - -/// The classic `histogram_quantile` aggregate: its `without` keys, `le` -/// column, and output column names. -fn classic_histogram(qe: &QueryExpr) -> (Vec, usize, Vec) { - let QueryExpr::Aggregate { - reduction: Reduction::Reduce(by), - measures, - .. - } = qe - else { - panic!("expected a reducing Aggregate, got {qe:?}"); - }; - let [AggIntent::HistogramQuantile { le, .. }] = measures.as_slice() else { - panic!("expected HistogramQuantile, got {measures:?}"); - }; - assert!(by.is_without(), "histogram_quantile groups without (le)"); - let names = qe - .output_schema() - .unwrap() - .fields - .iter() - .map(|c| c.name.clone()) - .collect(); - (by.keys().to_vec(), *le, names) -} - -// A classic histogram_quantile groups `without (le)` and names the child's -// `le` column, even when no matcher or grouping mentions `le`. -#[test] -fn classic_histogram_quantile_groups_without_le() { - let qe = lower("histogram_quantile(0.9, rate(http_duration_seconds_bucket[5m]))"); - let (keys, le, names) = classic_histogram(&qe); - let QueryExpr::Aggregate { child, .. } = &qe else { - unreachable!() - }; - let child = child.output_schema().unwrap(); - assert_eq!(child.fields[le].name, "le"); - assert_eq!(keys, vec![le]); - assert_eq!(names, vec!["histogram_quantile"]); -} - -// An explicit `sum by (le, job)` argument keeps `job` and drops `le` and the -// renamed sample value from the output labels. -#[test] -fn classic_histogram_quantile_over_sum_by_keeps_other_labels() { - let qe = - lower("histogram_quantile(0.9, sum by (le, job) (rate(http_duration_seconds_bucket[5m])))"); - let (keys, le, names) = classic_histogram(&qe); - // `sum by (le, job)` outputs `[job, le, sum]`. - assert_eq!((keys, le), (vec![1], 1)); - assert_eq!(names, vec!["job", "histogram_quantile"]); -} - -// Out-of-range and NaN quantiles lower unchanged; execution returns -Inf/+Inf/NaN. -#[test] -fn classic_histogram_quantile_keeps_out_of_range_quantiles() { - for (query, expected) in [ - ("histogram_quantile(-1, x_bucket)", -1.), - ("histogram_quantile(2, x_bucket)", 2.), - ] { - let QueryExpr::Aggregate { measures, .. } = lower(query) else { - panic!("{query}"); - }; - assert!( - matches!(measures.as_slice(), [AggIntent::HistogramQuantile { q, .. }] if *q == expected) - ); - } - let QueryExpr::Aggregate { measures, .. } = lower("histogram_quantile(NaN, x_bucket)") else { - panic!("NaN"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::HistogramQuantile { q, .. }] if q.is_nan())); -} - -// An argument whose closed output lacks `le` has no buckets. Prometheus -// returns an empty vector; lowering rejects it rather than guess a column. -#[test] -fn classic_histogram_quantile_rejects_an_argument_without_le() { - let error = lower_promql( - "histogram_quantile(0.9, sum by (job) (rate(x_bucket[5m])))", - AccuracyTarget::Exact, - ) - .unwrap_err(); - assert!(error.to_string().contains("le"), "{error}"); -} - -// ── rate / increase carry their own window (no Window node) ───────────────────── - -#[test] -fn rate_has_time_range_child_not_window() { - let qe = lower("rate(http_requests_total[5m])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate for rate, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Rate])); - let QueryExpr::TimeRange { range, .. } = child.as_ref() else { - panic!("expected TimeRange child (not Window), got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(300)); -} - -#[test] -fn increase_maps_to_increase_intent() { - let qe = lower("increase(errors_total[1h])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate for increase, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Increase])); - let QueryExpr::TimeRange { range, .. } = child.as_ref() else { - panic!("expected TimeRange child, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(3600)); -} - -// ── outer aggregation over an inner range-vector func is two levels ───────────── - -#[test] -fn sum_over_rate_keeps_both_levels() { - // Regression: `sum(rate(m[w]))` — the most common PromQL shape — must keep - // the cross-series Sum, not collapse to a bare per-series Rate. - let qe = lower("sum(rate(http_requests_total[5m]))"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate{{Sum}}, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected inner Aggregate{{Rate}}, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Rate])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -#[test] -fn sum_by_over_rate_groups_the_outer_sum() { - // `sum by (job) (rate(...))`: the grouping belongs to the OUTER sum and lands - // on a positional `Aggregate.by` (the same shape SQL produces) over the - // label-preserving inner Rate. Leaf = [ts, value, job] → by = [2]. - let qe = lower("sum by (job) (rate(http_requests_total[5m]))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate grouped by job, got {qe:?}"); - }; - assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!(matches!( - child.as_ref(), - QueryExpr::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate]) - )); -} - -#[test] -fn count_over_rate_keeps_both_levels() { - // The `Outer::Count` sibling of the `sum(rate(...))` bug. - let qe = lower("count(rate(http_requests_total[5m]))"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected outer Aggregate{{Count}}, got {qe:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); - assert!(matches!( - child.as_ref(), - QueryExpr::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate]) - )); -} - -#[test] -fn count_over_distinct_over_time_preserves_both_aggregates() { - // One series with window samples [1, 2] produces one distinct-count - // result (value 2). The outer count counts that one series, yielding 1. - for (query, reduction) in [ - ( - "count(distinct_over_time(unique_users[5m]))", - Reduction::by(vec![]), - ), - ( - "count by (job) (distinct_over_time(unique_users[5m]))", - Reduction::by(vec![2]), - ), - ] { - let dag = lower(query); - let QueryExpr::Aggregate { - measures, - reduction: actual, - child, - .. - } = &dag - else { - panic!("expected outer Count: {dag:?}"); - }; - assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), - "{query}: {dag:?}" - ); - assert_eq!(actual, &reduction, "{query}"); - let QueryExpr::Aggregate { - measures, - reduction, - child, - .. - } = child.as_ref() - else { - panic!("expected inner per-series Cardinality: {dag:?}"); - }; - assert!( - matches!(measures.as_slice(), [AggIntent::Cardinality { .. }]), - "{query}: {dag:?}" - ); - assert_eq!(reduction, &Reduction::PerEntity, "{query}"); - assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { range, .. } if range.as_secs() == 300) - ); - } -} - -// ── count / cardinality ─────────────────────────────────────────────────────── - -// Both selector fast paths and recursive vector expressions count rows, not values. -#[test] -fn count_never_lowers_to_distinct_sample_values() { - for query in [ - "count(up)", - "count by (job) (up)", - "count without (instance) (up)", - "count(up + 1)", - "count(count_over_time(up[5m]))", - "count_over_time(up[5m])", - ] { - let dag = lower(query); - let intents = all_intents(&dag); - assert!( - intents.iter().any(|i| matches!(i, AggIntent::Count { .. })), - "{query}: {dag:?}" - ); - assert!( - !intents - .iter() - .any(|i| matches!(i, AggIntent::Cardinality { .. })), - "{query}: {dag:?}" - ); - } -} - -#[test] -fn count_over_time_is_count_intent() { - let qe = lower("count_over_time(m[5m])"); - let QueryExpr::Aggregate { - measures, child, .. - } = &qe - else { - panic!("expected Aggregate"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -#[test] -fn outer_count_counts_series() { - // `count by (symbol) (count_over_time(...))`: inner per-series sample count - // over the window (label-preserving), outer cross-series row count grouped - // on a positional `Aggregate.by`. Leaf = [ts, value, symbol] → symbol = col 2. - let qe = lower("count by (symbol) (count_over_time(financial_last_trade_price[5m]))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected outer Aggregate grouped by symbol, got {qe:?}"); - }; - assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); - // Inner: Aggregate{Count} over TimeRange (per-series count_over_time). - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected Aggregate (count_over_time) under the outer count, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -// ── topk / bottomk ──────────────────────────────────────────────────────────── - -#[test] -fn topk_over_count_is_heavy_hitter_topk() { - let qe = lower(r#"topk by (service) (10, count_over_time(requests{env="prod"}[1m]))"#); - // Heavy-hitter: Aggregate{TopK} with grouping resolved to positional ids. - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected Aggregate with TopK, got {qe:?}"); - }; - // `service` is the only group key → resolved to a positional ColumnId. - assert_eq!(reduction.expect_reduce().len(), 1); - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 10, .. }] - )); - // The count_over_time under the TopK is a TimeRange-backed aggregate. - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected Aggregate (count_over_time) under TopK, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); - let QueryExpr::TimeRange { range, child } = child.as_ref() else { - panic!("expected TimeRange under Count aggregate, got {child:?}"); - }; - assert_eq!(*range, Duration::from_secs(60)); - assert!(matches!(child.as_ref(), QueryExpr::Scan { .. })); -} - -#[test] -fn topk_over_sum_is_value_weighted_heavy_hitter_topk() { - let qe = lower(r#"topk by (service) (5, sum_over_time(requests{env="prod"}[1m]))"#); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected Aggregate with TopK, got {qe:?}"); - }; - assert_eq!(reduction.expect_reduce().len(), 1); - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 5, .. }] - )); - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected Aggregate (sum_over_time) under TopK, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -#[test] -fn topk_over_avg_is_generic_sort_limit() { - let qe = lower("topk by (host) (5, avg_over_time(cpu[5m]))"); - let QueryExpr::Limit { n, offset, child } = &qe else { - panic!("expected Limit, got {qe:?}"); - }; - assert_eq!(*n, 5); - assert_eq!(*offset, 0); - let QueryExpr::Sort { - keys, - partition_by, - child, - } = child.as_ref() - else { - panic!("expected Sort under Limit, got {child:?}"); - }; - assert_eq!(keys.len(), 1); - assert!(!keys[0].ascending, "topk ranks descending"); - // `by (host)` is per-group ranking → it rides on `Sort.partition_by` - // (positional), not a `Partition` node (issue #12). `host` is col 2 in - // the per-series avg schema [ts, value, host]. - assert_eq!(partition_by, &vec![2]); - // Underneath: the label-preserving windowed avg aggregate (by: []), no - // intervening Partition. - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { reduction, measures, .. } - if reduction == &Reduction::PerEntity && matches!(measures.as_slice(), [AggIntent::Avg { .. }])), - "expected bare per-series Avg aggregate under Sort, got {child:?}" - ); -} - -#[test] -fn ungrouped_topk_over_sum_is_heavy_hitter() { - let qe = lower("topk(5, sum_over_time(m[5m]))"); - assert!(matches!(&qe, QueryExpr::Aggregate { .. })); - assert!(has_intent(&qe, |i| matches!(i, AggIntent::Sum { .. }))); - assert!(has_intent(&qe, |i| matches!( - i, - AggIntent::TopK { k: 5, .. } - ))); -} - -#[test] -fn bottomk_over_count_is_generic_sort_ascending() { - // `bottomk` is never a heavy-hitter (descending=false), even over count. - let qe = lower("bottomk(3, count_over_time(m[5m]))"); - let QueryExpr::Limit { n, child, .. } = &qe else { - panic!("expected Limit, got {qe:?}"); - }; - assert_eq!(*n, 3); - let QueryExpr::Sort { keys, .. } = child.as_ref() else { - panic!("expected Sort"); - }; - assert!(keys[0].ascending, "bottomk ranks ascending"); - // Count intent is still present (as the inner aggregate), no TopK. - assert!(has_intent(&qe, |i| matches!(i, AggIntent::Count { .. }))); - assert!(!has_intent(&qe, |i| matches!(i, AggIntent::TopK { .. }))); -} - -#[test] -fn bottomk_is_always_generic_sort_ascending() { - let qe = lower("bottomk(3, count_over_time(m[5m]))"); - let QueryExpr::Limit { n, child, .. } = &qe else { - panic!("expected Limit, got {qe:?}"); - }; - assert_eq!(*n, 3); - let QueryExpr::Sort { keys, .. } = child.as_ref() else { - panic!("expected Sort"); - }; - assert!(keys[0].ascending, "bottomk ranks ascending"); -} - -#[test] -fn topk_count_output_schema_carries_group_key() { - // The inner Count is per-series (label-preserving), so the group-by key - // (`service`) flows through to the outer TopK's `by` column. Leaf schema = - // [ts, value, service] → TopK groups on service (col 2). - let qe = lower("topk by (service) (5, count_over_time(m[1m]))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected Aggregate{{TopK}}, got {qe:?}"); - }; - assert_eq!( - reduction, - &Reduction::by(vec![2]), - "service is col 2 in [ts, value, service]" - ); - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 5, .. }] - )); - // Inner Count aggregate is visible with its TimeRange child. - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected inner Aggregate{{Count}}, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); - assert!(matches!(child.as_ref(), QueryExpr::TimeRange { .. })); -} - -// ── binary ops ──────────────────────────────────────────────────────────────── - -#[test] -fn binary_op_division() { - let qe = lower("rate(a[5m]) / rate(b[5m])"); - let QueryExpr::BinaryOp { op, lhs, rhs, .. } = &qe else { - panic!("expected BinaryOp, got {qe:?}"); - }; - assert_eq!(*op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)); - assert!( - matches!(lhs.as_ref(), QueryExpr::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate])) - ); - assert!( - matches!(rhs.as_ref(), QueryExpr::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate])) - ); -} - -#[test] -fn binary_op_with_on_grouping() { - let qe = lower("a / on(host) b"); - let QueryExpr::BinaryOp { vector_match, .. } = &qe else { - panic!("expected BinaryOp, got {qe:?}"); - }; - let vm = vector_match.as_ref().expect("vector_match present"); - use asap_types::pre_asap::VectorMatchKind; - assert_eq!(vm.kind, VectorMatchKind::On); - assert_eq!(vm.labels, vec!["host".to_string()]); -} - -// `bool` changes a comparison from a filter to a 0/1 result, so the IR must -// carry it. -#[test] -fn bool_comparisons_are_distinct() { - let op = |q: &str| match lower(q) { - QueryExpr::BinaryOp { op, .. } => op, - other => panic!("expected BinaryOp, got {other:?}"), - }; - assert_eq!(op("a > 1"), BinaryOpKind::Compare(CompareOpKind::Gt)); - assert_eq!( - op("a > bool 1"), - BinaryOpKind::CompareBool(CompareOpKind::Gt) - ); - assert_eq!( - op("a == bool on(job) b"), - BinaryOpKind::CompareBool(CompareOpKind::Eq) - ); -} - -#[test] -fn binary_op_binds_each_branch_against_its_own_schema() { - // Each side scans a different metric and groups by a different label. With a - // single root schema threaded to both branches, the left scan would leak the - // right's group key (and vice-versa). Per-branch binding keeps them separate. - let qe = lower("count by (job) (a) / count by (region) (b)"); - let QueryExpr::BinaryOp { lhs, rhs, .. } = &qe else { - panic!("expected BinaryOp, got {qe:?}"); - }; - let lcols = scan_columns(lhs); - let rcols = scan_columns(rhs); - assert!( - lcols.iter().any(|c| c == "job") && !lcols.iter().any(|c| c == "region"), - "lhs scan schema leaked the rhs key: {lcols:?}" - ); - assert!( - rcols.iter().any(|c| c == "region") && !rcols.iter().any(|c| c == "job"), - "rhs scan schema leaked the lhs key: {rcols:?}" - ); -} - -/// Collect every `AggIntent` in the DAG, root-to-leaf. -fn all_intents(e: &QueryExpr) -> Vec { - let mut out = Vec::new(); - collect_intents(e, &mut out); - out -} - -fn collect_intents(e: &QueryExpr, out: &mut Vec) { - match e { - QueryExpr::Aggregate { - measures, child, .. - } => { - out.extend(measures.iter().cloned()); - collect_intents(child, out); - } - QueryExpr::TimeRange { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => collect_intents(child, out), - QueryExpr::BinaryOp { lhs, rhs, .. } => { - collect_intents(lhs, out); - collect_intents(rhs, out); - } - _ => {} - } -} - -/// True if any `AggIntent` anywhere in the DAG satisfies `pred`. -fn has_intent bool>(e: &QueryExpr, pred: F) -> bool { - all_intents(e).iter().any(pred) -} - -/// Field names on the first `Scan` reachable by descending single-child nodes. -fn scan_columns(e: &QueryExpr) -> Vec { - match e { - QueryExpr::Scan { schema, .. } => schema.fields.iter().map(|c| c.name.clone()).collect(), - QueryExpr::Aggregate { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => scan_columns(child), - _ => vec![], - } -} - -// ── without(...) grouping (issue #39) ─────────────────────────────────────────── - -#[test] -fn without_grouping_lowers_to_the_exclusion_form() { - // `sum without (instance) (rate(m[5m]))` — a cross-series reduction over the - // per-series rate, grouped by every label except `instance`. The excluded - // label is stored positionally (the SchemaResolver seeds it), the grouping is the - // `without` form, and the output schema stays open. - let qe = lower("sum without (instance) (rate(m[5m]))"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &qe - else { - panic!("expected an Aggregate, got {qe:?}"); - }; - let by = reduction.expect_reduce(); - assert!(by.is_without()); - assert_eq!(by.keys().len(), 1, "excluded `instance`"); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - // The inner per-series rate is preserved (label-preserving) under the outer - // cross-series `without` reduction. - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Rate])) - ); - assert!(!qe.output_schema().unwrap().closed); -} - -// ── parameter validation (reject rather than silently truncate/garble) ────────── - -#[test] -fn fractional_or_negative_topk_k_is_rejected() { - // `as u64` would silently truncate 2.7→2 / saturate -1→0. - assert!(lower_promql("topk(2.7, count_over_time(m[1m]))", AccuracyTarget::Exact).is_err()); - assert!(lower_promql("bottomk(2.5, sum_over_time(m[1m]))", AccuracyTarget::Exact).is_err()); -} - -#[test] -fn out_of_range_quantile_phi_is_accepted() { - // Prometheus defines out-of-range phi results; lowering must preserve it. - for query in [ - "quantile(1.5, up)", - "quantile_over_time(1.5, m[5m])", - "histogram_quantile(2.0, rate(b[5m]))", - ] { - assert!( - lower_promql(query, AccuracyTarget::Exact).is_ok(), - "{query}" - ); - } -} - -#[test] -fn function_wrapped_range_vector_is_rejected_not_stripped() { - // `rate(abs(m[5m]))` must NOT silently lower as `rate(m[5m])` — the wrapper - // is rejected (here, at parse or in extract_matrix), never stripped. - assert!( - lower_promql("rate(abs(http_requests_total[5m]))", AccuracyTarget::Exact).is_err(), - "function-wrapped range vector should be rejected" - ); -} - -#[test] -fn pathologically_nested_query_is_rejected_not_stack_overflow() { - // 300 nested parens parse fine but exceed the walker's depth limit (256); - // it must return an error, not overflow the stack. - let q = format!("{}m{}", "(".repeat(300), ")".repeat(300)); - let err = lower_promql(&q, AccuracyTarget::Exact).unwrap_err(); - assert!(format!("{err}").contains("nesting"), "got {err}"); -} - -// Behavior: every parser-accepted `fill` modifier form is rejected with a -// fill-specific lowering error rather than silently dropped. -#[test] -fn fill_modifiers_are_rejected_not_ignored() { - for q in [ - "a + fill(0) b", - "a + fill_left(1) b", - "a + fill_right(2) b", - "a + fill_left(1) fill_right(2) b", - "a + fill_right(2) fill_left(1) b", - "a + on(job) fill(0) b", - "a * ignoring(instance) group_left(env) fill_right(0) b", - "a > bool on(job) fill(0) b", - "sum(a - on(job) group_right fill_left(0) b)", - ] { - match lower_promql(q, AccuracyTarget::Exact) { - Err(LoweringError::UnsupportedFeature(m)) if m.contains("`fill`") => {} - other => panic!("expected fill rejection for {q:?}, got {other:?}"), - } - } -} - -// ── accuracy propagation ────────────────────────────────────────────────────── - -#[test] -fn accuracy_target_flows_into_quantile_intent() { - let qe = lower_promql( - "quantile_over_time(0.9, m[5m])", - AccuracyTarget::Epsilon(0.01), - ) - .unwrap(); - let QueryExpr::Aggregate { measures, .. } = &qe else { - panic!("expected Aggregate"); - }; - assert!(matches!( - &measures[0], - AggIntent::Quantile { accuracy: AccuracyTarget::Epsilon(e), .. } if (*e - 0.01).abs() < 1e-12 - )); -} - -// ── schema flow (positional, carried on Scan; derived on demand) ───────────────── - -#[test] -fn aggregate_output_schema_preserves_time_axis_and_labels() { - let qe = lower(r#"quantile_over_time(0.99, http_request_duration{env="prod"}[5m])"#); - // Per-series reduction: the root is Aggregate { TimeRange { Scan } }. - // The SchemaResolver adds all referenced label names (group keys AND filter - // predicate columns) to the scan schema, so `env` appears as a column - // even though it is only used as a filter. - // per_series_reduction_schema preserves the time axis and all label columns. - let QueryExpr::Aggregate { .. } = &qe else { - panic!("expected Aggregate, got {qe:?}"); - }; - let schema = qe.output_schema().expect("aggregate schema"); - let names: Vec<&str> = schema.fields.iter().map(|c| c.name.as_str()).collect(); - assert_eq!(names, vec!["ts", "value", "env"]); - assert_eq!( - schema.time_index, - Some(0), - "per-series over_time preserves the time axis" - ); -} - -#[test] -fn scan_schema_carries_ts_value_and_group_keys() { - // `service` is a group key → the SchemaResolver lands it in the self-contained - // Scan schema (positional). `env` is only a filter, so it is not a column. - let qe = lower("count by (service) (count_over_time(requests[1m]))"); - fn find_scan(n: &QueryExpr) -> &QueryExpr { - match n { - QueryExpr::Scan { .. } => n, - QueryExpr::TimeRange { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Filter { child, .. } => find_scan(child), - other => panic!("unexpected node {other:?}"), - } - } - let QueryExpr::Scan { schema, .. } = find_scan(&qe) else { - unreachable!() - }; - let mut names: Vec<&str> = schema.fields.iter().map(|c| c.name.as_str()).collect(); - names.sort(); - assert_eq!(names, vec!["service", "ts", "value"]); - assert_eq!(schema.time_index, Some(0)); // ts -} - -// ── batch entry point ───────────────────────────────────────────────────────── - -#[test] -fn batch_lowers_each_entry_and_reads_per_query_accuracy() { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![ - BatchEntry { - query: Query("rate(a[5m])".into()), - requirements: QueryRequirements::default(), - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }, - BatchEntry { - query: Query("quantile_over_time(0.9, b[5m])".into()), - requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(AccuracyTarget::Epsilon(0.02)), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }, - ]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload { - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - ..Default::default() - }, - ..Default::default() - }), - }; - let results = lower_promql_workload(&workload, 0).expect("valid workload"); - assert_eq!(results.len(), 2); -} - -#[test] -fn batch_rejects_non_promql_language() { - use asap_types::workload::SqlDialect; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: Some(vec![BatchEntry { - query: Query("SELECT 1".into()), - requirements: QueryRequirements::default(), - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }]), - repeating_queries: None, - }, - data_workload: None, - }; - assert!(matches!( - lower_promql_workload(&workload, 0), - Err(LoweringError::WrongLanguage(_)) - )); -} - -// ── #12: one home per grouping concept (the canonical `Partition` node is removed) ── -// -// `Partition` and `Aggregate.by` were two ways to express grouping. #12 collapses -// them: a reducing GROUP BY → `Aggregate.by`; per-group *ranking* (split without -// reduce) → `Sort.partition_by`; parallel sharding → a deployment's own -// physical stage. There is no longer a canonical `Partition` node. These -// tests pin both surviving canonical homes. - -#[test] -fn reducing_group_by_lowers_to_aggregate_by() { - // Cross-series reduce, no keys → bare `Aggregate { reduction: Reduce([]) }`. - let q = lower("sum(http_requests_total)"); - assert!( - matches!(q, QueryExpr::Aggregate { ref reduction, .. } if reduction == &Reduction::by(vec![])) - ); - - // Cross-series reduce grouped by a label → `Aggregate.reduction`. - let q = lower("sum by (job) (http_requests_total)"); - assert!(matches!(q, QueryExpr::Aggregate { ref reduction, .. } - if reduction.expect_reduce().len() == 1)); - - // Reduce over a label-preserving `rate` grouped by a label → still - // `Aggregate.reduction` (the keys resolve against rate's preserved schema). - let q = lower("sum by (job) (rate(http_requests_total[5m]))"); - assert!(matches!(q, QueryExpr::Aggregate { ref reduction, .. } - if reduction.expect_reduce().len() == 1)); -} - -#[test] -fn generic_topk_grouping_lowers_to_sort_partition_by() { - // Per-group ranking (`topk by (host)`, non-heavy-hitter) groups *without* - // reducing → the grouping rides on `Sort.partition_by`, and the windowed - // reduction beneath stays label-preserving (`by: []`). No `Partition` node. - let q = lower("topk by (host) (5, avg_over_time(cpu[5m]))"); - let QueryExpr::Limit { child, .. } = &q else { - panic!("expected Limit, got {q:?}"); - }; - let QueryExpr::Sort { - partition_by, - child, - .. - } = child.as_ref() - else { - panic!("expected Sort, got {child:?}"); - }; - assert_eq!(partition_by, &vec![2], "host is col 2 in [ts, value, host]"); - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { reduction, .. } if reduction == &Reduction::PerEntity) - ); -} - -#[test] -fn topk_over_bare_selector_by_label_ranks_per_group() { - // `topk(3, http_requests_total) by (job)` — top-3 series per `job`. A bare - // instant selector ranks its OWN samples; it must not be wrapped in an - // implicit cross-series `Sum`, which would collapse the `job` partition - // label before `Sort.partition_by` resolves it (issue #30 — follow-up to the - // Partition→Sort.partition_by reframe in #12). Expected: - // Limit{3} → Sort{value desc, partition_by:[job]} → Scan - let q = lower("topk(3, http_requests_total) by (job)"); - let QueryExpr::Limit { n, child, .. } = &q else { - panic!("expected Limit, got {q:?}"); - }; - assert_eq!(*n, 3); - let QueryExpr::Sort { - keys, - partition_by, - child, - } = child.as_ref() - else { - panic!("expected Sort, got {child:?}"); - }; - assert!(!keys[0].ascending, "topk ranks descending"); - assert_eq!(partition_by, &vec![2], "job is col 2 in [ts, value, job]"); - // No implicit reducing aggregate — the selector is label-preserving, so the - // sort is directly over the selector horizon (the `job` label survives to partition by). - assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Scan { .. })), - "ranking is over the bare selector horizon, not a reducing Aggregate, got {child:?}" - ); - assert!( - !has_intent(&q, |i| matches!(i, AggIntent::Sum { .. })), - "no implicit Sum is introduced over a bare selector" - ); -} - -#[test] -fn topk_over_bare_selector_ranks_raw_samples() { - // Even without `by`, `topk(3, m)` ranks the raw instant-vector samples — it - // does not sum them. The sort sits directly over the Scan, partition empty. - let q = lower("topk(3, http_requests_total)"); - let QueryExpr::Limit { child, .. } = &q else { - panic!("expected Limit, got {q:?}"); - }; - let QueryExpr::Sort { - partition_by, - child, - .. - } = child.as_ref() - else { - panic!("expected Sort, got {child:?}"); - }; - assert!(partition_by.is_empty(), "no `by` → global ranking"); - assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Scan { .. })) - ); - assert!(!has_intent(&q, |i| matches!(i, AggIntent::Sum { .. }))); -} - -// ── Issue #109: histogram_quantiles fans out into one branch per φ ────────── - -/// The `(label value, intent)` of each `histogram_quantiles` branch. -fn quantile_branches(q: &QueryExpr) -> Vec<(String, AggIntent)> { - let QueryExpr::Concat { children, .. } = q else { - panic!("expected a Concat at the root, got {q:?}"); - }; - children - .iter() - .map(|c| { - let QueryExpr::PromqlRelabel { value, child, .. } = c else { - panic!("expected PromqlRelabel per branch, got {c:?}"); - }; - let QueryExpr::Literal(ScalarValue::Utf8(v)) = value.as_ref() else { - panic!("expected a literal label value, got {value:?}"); - }; - let QueryExpr::Aggregate { measures, .. } = child.as_ref() else { - panic!("expected an Aggregate under the PromqlRelabel, got {child:?}"); - }; - (v.clone(), measures[0].clone()) - }) - .collect() -} - -#[test] -fn histogram_quantiles_fans_out_over_native_histograms() { - // Raw / native-histogram argument → the sketch-able `Quantile` intent, - // exactly as the single-quantile `histogram_quantile` would choose. - let q = lower(r#"histogram_quantiles(testhistogram3, "q", 0, 0.25, 1)"#); - let branches = quantile_branches(&q); - assert_eq!(branches.len(), 3); - let labels: Vec<_> = branches.iter().map(|(l, _)| l.as_str()).collect(); - assert_eq!( - labels, - ["0.0", "0.25", "1.0"], - "OpenMetrics float formatting" - ); - for (_, intent) in &branches { - assert!( - matches!(intent, AggIntent::Quantile { .. }), - "native histogram → sketch-able Quantile, got {intent:?}" - ); - } -} - -#[test] -fn histogram_quantiles_over_classic_buckets_interpolates() { - // `_bucket` argument → exact cumulative-bucket interpolation, never a sketch. - let q = lower(r#"histogram_quantiles(request_duration_seconds_bucket, "q", 0.5, 0.9)"#); - for (_, intent) in quantile_branches(&q) { - assert!( - matches!(intent, AggIntent::HistogramQuantile { .. }), - "classic buckets → HistogramQuantile, got {intent:?}" - ); - } -} - -#[test] -fn histogram_quantiles_branches_are_union_compatible() { - // `Concat` derives its schema from the first child, so every branch must - // agree on column names — the φ lives in the label, not the column name. - let q = lower(r#"histogram_quantiles(testhistogram3, "q", 0.5, 0.9)"#); - let QueryExpr::Concat { children, .. } = &q else { - panic!("expected Concat"); - }; - let shapes: Vec> = children - .iter() - .map(|c| { - c.output_schema() - .expect("branch schema") - .fields - .iter() - .map(|c| c.name.clone()) - .collect() - }) - .collect(); - assert_eq!(shapes[0], shapes[1], "branches must be union-compatible"); - assert_eq!(shapes[0], vec!["value".to_string(), "q".to_string()]); - assert_eq!( - q.output_schema().expect("merged schema").fields.len(), - 2, - "the merged schema describes every branch" - ); -} - -#[test] -fn histogram_quantiles_uses_the_given_label_name() { - let q = lower(r#"histogram_quantiles(h, "phi", 0.5)"#); - let QueryExpr::Concat { children, .. } = &q else { - panic!("expected Concat"); - }; - let QueryExpr::PromqlRelabel { dst, .. } = &children[0] else { - panic!("expected PromqlRelabel"); - }; - assert_eq!(dst, "phi"); -} - -#[test] -fn histogram_quantiles_formats_small_quantiles_like_prometheus() { - // `labels.FormatOpenMetricsFloat`: Go's %g, so exponent form below 1e-4. - let q = lower(r#"histogram_quantiles(h, "q", 0.00001)"#); - assert_eq!(quantile_branches(&q)[0].0, "1e-05"); -} - -#[test] -fn histogram_quantiles_rejects_an_out_of_range_quantile() { - // Same rule as `histogram_quantile(φ, …)` — one bad φ fails the whole call. - for q in [ - r#"histogram_quantiles(h, "q", -0.1)"#, - r#"histogram_quantiles(h, "q", 1.01)"#, - r#"histogram_quantiles(h, "q", 0.5, NaN)"#, - ] { - assert!( - lower_promql(q, AccuracyTarget::Exact).is_err(), - "{q} should be rejected" - ); - } -} - -// A subquery's `offset`/`@` shift the whole subquery, so the DAG keeps them. -#[test] -fn subquery_time_shift_is_retained() { - let QueryExpr::Aggregate { child, .. } = lower("max_over_time(m[5m:1m] offset 1m)") else { - panic!("expected a range function"); - }; - let QueryExpr::TimeShift { shift, child } = child.as_ref() else { - panic!("subquery offset was dropped: {child:?}"); - }; - assert_eq!(shift.offset_ms, 60_000); - assert!(matches!(child.as_ref(), QueryExpr::PromqlSubquery { .. })); - assert!(matches!( - lower("max_over_time(m[5m:1m] @ 100)"), - QueryExpr::Aggregate { child, .. } if matches!(child.as_ref(), QueryExpr::TimeShift { .. }) - )); -} diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs deleted file mode 100644 index 1f15b1ca2..000000000 --- a/crates/frontend-promql/tests/support.rs +++ /dev/null @@ -1,52 +0,0 @@ -use asap_frontend_promql::{ - lower_promql_workload, lower_promql_workload_with_histograms, HistogramCatalog, PromqlError, -}; -use asap_types::pre_asap::QueryExpr; -use asap_types::types::AccuracyTarget; -use asap_types::workload::{ - AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, PlanningWorkload, - Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, TimeSelection, -}; - -fn workload(query: &str, accuracy: AccuracyTarget) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(accuracy), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - time_selection: TimeSelection::default(), - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload { - data_ingestion_interval: Evidence { - value: Some(DurationMs(1_000)), - ..Default::default() - }, - ..Default::default() - }), - } -} - -pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Result { - let mut lowered = lower_promql_workload(&workload(query, accuracy), 0)?; - Ok(lowered.remove(0)) -} - -#[allow(dead_code)] -pub fn lower_promql_with_histograms( - query: &str, - accuracy: AccuracyTarget, - histograms: HistogramCatalog, -) -> Result { - let mut lowered = - lower_promql_workload_with_histograms(&workload(query, accuracy), histograms, 0)?; - Ok(lowered.remove(0)) -} diff --git a/crates/frontend-sql/src/error.rs b/crates/frontend-sql/src/error.rs deleted file mode 100644 index 11d5acff4..000000000 --- a/crates/frontend-sql/src/error.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::fmt; - -use asap_types::pre_asap::ResolveDAGError; - -/// Errors from lowering a SQL query (parse + plan via DataFusion → the -/// canonical, unresolved DAG, built directly → -/// [`resolve_root`](asap_types::pre_asap::resolve_root) binds it to the -/// resolved DAG, issue #179). -/// -/// Carries no PromQL type — the SQL front end never depends on the PromQL -/// parser. The language-neutral variants (`UnsupportedFeature` / `WrongLanguage` -/// / `Convert`) are mirrored by [`asap_frontend_promql::PromqlError`] rather -/// than shared, so neither front end pulls the other's parser. -#[derive(Debug)] -pub enum SqlError { - /// DataFusion failed to parse / plan the SQL query. - DataFusion(datafusion::error::DataFusionError), - /// A table referenced by the query is absent from the catalog. - TableNotFound(String), - /// A SQL aggregate function not supported in this version. - UnsupportedAggregate(String), - /// A SQL scalar expression that could not be lowered. - InvalidExpression(String), - /// The SQL dialect is not supported (only DataFusionSQL is implemented). - UnsupportedDialect(String), - /// A structural feature (JOIN type / subquery / derived table) not - /// supported in this version. - UnsupportedFeature(String), - /// The workload's query language is not SQL. - WrongLanguage(String), - /// Resolving the canonical unresolved DAG failed (name resolution - /// against the bound schema). - Convert(ResolveDAGError), -} - -impl fmt::Display for SqlError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::DataFusion(e) => write!(f, "DataFusion error: {e}"), - Self::TableNotFound(t) => write!(f, "table not found in catalog: {t}"), - Self::UnsupportedAggregate(n) => write!(f, "unsupported aggregate: {n}"), - Self::InvalidExpression(m) => write!(f, "invalid expression: {m}"), - Self::UnsupportedDialect(d) => write!(f, "unsupported SQL dialect: {d}"), - Self::UnsupportedFeature(m) => write!(f, "unsupported feature: {m}"), - Self::WrongLanguage(l) => write!(f, "unsupported query language: {l}"), - Self::Convert(e) => write!(f, "column resolution failed: {e}"), - } - } -} - -impl std::error::Error for SqlError {} - -impl From for SqlError { - fn from(e: ResolveDAGError) -> Self { - Self::Convert(e) - } -} - -impl From for SqlError { - fn from(mut e: datafusion::error::DataFusionError) -> Self { - use datafusion::error::DataFusionError; - // Report the first underlying `Plan`/`SQL`/... error. The planner - // collects every error it finds and wraps each with a source-span - // diagnostic; neither carries anything this front end reports. - loop { - e = match e { - DataFusionError::Diagnostic(_, inner) => *inner, - DataFusionError::Collection(errors) if !errors.is_empty() => { - errors.into_iter().next().unwrap() - } - other => break Self::DataFusion(other), - }; - } - } -} diff --git a/crates/frontend-sql/src/lib.rs b/crates/frontend-sql/src/lib.rs deleted file mode 100644 index aa1449e5e..000000000 --- a/crates/frontend-sql/src/lib.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! SQL front end: parse + plan (via DataFusion) → the canonical, unresolved -//! shape, built directly (issue #179) → [`resolve_root`]. -//! -//! Emits [`UnresolvedQueryExpr`](asap_types::pre_asap::UnresolvedQueryExpr) itself — the -//! canonical `QueryExpr`, generic over an unresolved -//! [`ColumnRef`](asap_types::pre_asap::ColumnRef) — directly, rather than a -//! separate per-language relational DAG; `resolve_root` runs the -//! [`SchemaResolver`](asap_types::pre_asap::SchemaResolver) for positional name resolution. -//! Depends on DataFusion only — never on the PromQL parser. - -pub mod error; -pub mod sql; - -use asap_types::pre_asap::resolve_root; -use asap_types::pre_asap::QueryExpr; -use asap_types::types::AccuracyTarget; -use asap_types::workload::{QueryLanguage, QueryWorkload, SqlDialect}; - -pub use error::SqlError; -pub use sql::{SqlCatalog, SqlLowerer}; - -/// Lower a single SQL query string to the canonical, resolved `QueryExpr`, -/// parsed as `SqlDialect::DataFusionSQL`. -/// -/// The `catalog` supplies table schemas (used both to plan the SQL with -/// DataFusion and to carry positional column identity into the resolved -/// DAG). `accuracy` is threaded onto every approximate intent as it's built. -pub async fn lower_sql( - query: &str, - catalog: &SqlCatalog, - accuracy: AccuracyTarget, -) -> Result { - lower_sql_dialect(query, catalog, SqlDialect::DataFusionSQL, accuracy).await -} - -/// Lower a single SQL query string under an explicit [`SqlDialect`]. -/// -/// `ClickhouseSQL` parses via sqlparser's vendored `ClickHouseDialect` -/// (array-lambda syntax, `arr[-1]` indexing). It also teaches DataFusion's -/// planner the ClickHouse-only builtin functions listed in -/// `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (`uniqExact`, `countIf`) -/// — every other ClickHouse-only builtin still fails to plan. -/// `ElasticSQL` has no vendored parser and always returns `UnsupportedDialect`. -pub async fn lower_sql_dialect( - query: &str, - catalog: &SqlCatalog, - dialect: SqlDialect, - accuracy: AccuracyTarget, -) -> Result { - let unresolved = SqlLowerer::with_dialect(catalog, dialect) - .lower(query, &accuracy) - .await?; - let resolved = resolve_root(&unresolved)?; - // Binding resolves names; schema inference also checks result types such - // as temporal subtraction, whose duration unit the IR cannot represent. - resolved - .output_schema() - .map_err(|error| SqlError::InvalidExpression(error.to_string()))?; - Ok(resolved) -} - -/// Lower every SQL batch entry in `workload` to a `QueryExpr`. -/// -/// One `Result` per entry — errors are per-query, not fatal for the batch. -/// Returns `WrongLanguage` for every entry if the workload is not SQL, and -/// `UnsupportedDialect` for `ElasticSQL` (no vendored parser). -pub async fn lower_sql_batch( - workload: &QueryWorkload, - catalog: &SqlCatalog, -) -> Vec> { - let entries = match &workload.query_batch { - Some(e) if !e.is_empty() => e, - _ => return vec![], - }; - - // `DataFusion` is a legacy alias for `SQL(DataFusionSQL)`; accept both. - if !matches!( - workload.language, - QueryLanguage::SQL(_) | QueryLanguage::DataFusion - ) { - let lang = format!("{:?}", workload.language); - return entries - .iter() - .map(|_| Err(SqlError::WrongLanguage(lang.clone()))) - .collect(); - } - let dialect = match &workload.language { - QueryLanguage::SQL(d) => d.clone(), - _ => SqlDialect::DataFusionSQL, - }; - if matches!(dialect, SqlDialect::ElasticSQL) { - return entries - .iter() - .map(|_| Err(SqlError::UnsupportedDialect("ElasticSQL".into()))) - .collect(); - } - - let mut results = Vec::with_capacity(entries.len()); - for entry in entries { - let accuracy = entry.requirements.accuracy.target(); - results.push(lower_sql_dialect(&entry.query.0, catalog, dialect.clone(), accuracy).await); - } - results -} - -/// Unified SQL lowering; promoted to the root API at the planner cutover. -pub mod unified; diff --git a/crates/frontend-sql/src/sql/clickhouse_ast.rs b/crates/frontend-sql/src/sql/clickhouse_ast.rs deleted file mode 100644 index f9246d22a..000000000 --- a/crates/frontend-sql/src/sql/clickhouse_ast.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! Structural ClickHouse syntax normalization before DataFusion type inference. -use datafusion::sql::sqlparser::ast::{ - visit_expressions, visit_expressions_mut, AccessExpr, BinaryOperator, Expr, Function, - FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, - ObjectNamePart, Query, SelectItem, SetExpr, Statement, Subscript, Value, VisitMut, VisitorMut, -}; -use std::ops::ControlFlow; - -use super::collection_planning::MAP_PLANNING_NAME; - -pub(super) fn normalize(statement: &mut Statement) { - struct PreserveNames; - impl VisitorMut for PreserveNames { - type Break = (); - fn pre_visit_query(&mut self, query: &mut Query) -> ControlFlow<()> { - fn preserve(body: &mut SetExpr) { - match body { - SetExpr::Select(select) => { - for item in &mut select.projection { - if let SelectItem::UnnamedExpr(expr) = item { - let mut changed = false; - let _: ControlFlow<()> = visit_expressions_mut(expr, |node| { - changed |= normalize_map_access(node); - ControlFlow::Continue(()) - }); - let _: ControlFlow<()> = visit_expressions(expr, |candidate| { - if let Expr::Function(function) = candidate { - changed |= - unquoted_name(&function.name).is_some_and(|name| { - matches!( - name.to_ascii_lowercase().as_str(), - "modulo" - | "map" - | "mapconcat" - | "arrayelement" - | "tupleelement" - ) - }); - } - ControlFlow::Continue(()) - }); - if changed { - let alias = Ident::with_quote('"', expr.to_string()); - let value = - std::mem::replace(expr, Expr::Value(Value::Null.into())); - *item = SelectItem::ExprWithAlias { expr: value, alias }; - } - } - } - } - SetExpr::SetOperation { left, right, .. } => { - preserve(left); - preserve(right); - } - _ => {} - } - } - preserve(&mut query.body); - ControlFlow::Continue(()) - } - } - let _: ControlFlow<()> = statement.visit(&mut PreserveNames); - let _: ControlFlow<()> = visit_expressions_mut(statement, |expr| { - normalize_map_access(expr); - let Expr::Function(function) = expr else { - return ControlFlow::Continue(()); - }; - if unquoted_name(&function.name).is_some_and(|name| name.eq_ignore_ascii_case("map")) { - function.name = ObjectName::from(vec![Ident::new(MAP_PLANNING_NAME)]); - return ControlFlow::Continue(()); - } - if !unquoted_name(&function.name).is_some_and(|name| name.eq_ignore_ascii_case("modulo")) - || !matches!(function.parameters, FunctionArguments::None) - || function.filter.is_some() - || function.over.is_some() - || function.null_treatment.is_some() - || !function.within_group.is_empty() - { - return ControlFlow::Continue(()); - } - let FunctionArguments::List(arguments) = &function.args else { - return ControlFlow::Continue(()); - }; - if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { - return ControlFlow::Continue(()); - } - let [FunctionArg::Unnamed(FunctionArgExpr::Expr(left)), FunctionArg::Unnamed(FunctionArgExpr::Expr(right))] = - arguments.args.as_slice() - else { - return ControlFlow::Continue(()); - }; - *expr = Expr::BinaryOp { - left: Box::new(left.clone()), - op: BinaryOperator::Modulo, - right: Box::new(right.clone()), - }; - ControlFlow::Continue(()) - }); -} - -/// The name of a single-part, unquoted function name such as `modulo`. -fn unquoted_name(name: &ObjectName) -> Option<&str> { - match name.0.as_slice() { - [ObjectNamePart::Identifier(ident)] if ident.quote_style.is_none() => Some(&ident.value), - _ => None, - } -} - -/// Rewrites a bracket-only access chain such as `m['k'][1]` into nested -/// `arrayElement` calls. Chains with a dot access or a slice stay unchanged. -fn normalize_map_access(expression: &mut Expr) -> bool { - let Expr::CompoundFieldAccess { access_chain, .. } = expression else { - return false; - }; - if access_chain.is_empty() - || access_chain - .iter() - .any(|access| !matches!(access, AccessExpr::Subscript(Subscript::Index { .. }))) - { - return false; - } - let Expr::CompoundFieldAccess { root, access_chain } = - std::mem::replace(expression, Expr::Value(Value::Null.into())) - else { - unreachable!() - }; - let mut input = *root; - for access in access_chain { - let AccessExpr::Subscript(Subscript::Index { index }) = access else { - unreachable!() - }; - input = Expr::Function(Function { - name: ObjectName::from(vec![Ident::new("arrayElement")]), - uses_odbc_syntax: false, - parameters: FunctionArguments::None, - args: FunctionArguments::List(FunctionArgumentList { - duplicate_treatment: None, - clauses: vec![], - args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr(input)), - FunctionArg::Unnamed(FunctionArgExpr::Expr(index)), - ], - }), - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - }); - } - *expression = input; - true -} diff --git a/crates/frontend-sql/src/sql/collection_planning.rs b/crates/frontend-sql/src/sql/collection_planning.rs deleted file mode 100644 index 9d28cd1df..000000000 --- a/crates/frontend-sql/src/sql/collection_planning.rs +++ /dev/null @@ -1,194 +0,0 @@ -//! DataFusion planning adapters. Types come from the canonical signature rules; -//! physical evaluation deliberately remains the query engine's responsibility. -use super::types::{arrow_to_dtype, dtype_to_arrow, scalar_value_to_asap}; -use asap_types::pre_asap::scalar_type_rules::{ - element_access_type, struct_field_type, MapScalarFunction, -}; -use asap_types::pre_asap::{Field, QueryExpr, Schema}; -use datafusion::arrow::datatypes::{DataType, Field as ArrowField, FieldRef}; -use datafusion::common::{DataFusionError, Result, ScalarValue as DfScalarValue}; -use datafusion::logical_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, - TypeSignature, Volatility, -}; -use datafusion::prelude::SessionContext; -use std::hash::{Hash, Hasher}; -use std::sync::Arc; - -/// The name ClickHouse's `map(...)` is planned under. DataFusion's SQL planner -/// reserves `map` for its own constructor (and rejects `map()`), so -/// `clickhouse_ast::normalize` renames the call and lowering restores `map`. -pub(super) const MAP_PLANNING_NAME: &str = "asap_map_construct"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PlanningFunction { - Map(MapScalarFunction), - Element, - StructField, -} - -pub(super) fn register(context: &SessionContext) { - for (name, function) in [ - ( - MAP_PLANNING_NAME, - PlanningFunction::Map(MapScalarFunction::Construct), - ), - ( - "mapconcat", - PlanningFunction::Map(MapScalarFunction::Concat), - ), - ("arrayelement", PlanningFunction::Element), - ("tupleelement", PlanningFunction::StructField), - ] { - context.register_udf(ScalarUDF::from(CollectionPlanningFunction { - name, - function, - signature: match function { - PlanningFunction::Map(MapScalarFunction::Construct) => Signature::one_of( - vec![TypeSignature::Exact(vec![]), TypeSignature::VariadicAny], - Volatility::Immutable, - ), - PlanningFunction::Map(MapScalarFunction::Access) - | PlanningFunction::Element - | PlanningFunction::StructField => Signature::any(2, Volatility::Immutable), - PlanningFunction::Map(MapScalarFunction::Concat) => { - Signature::variadic_any(Volatility::Immutable) - } - }, - })); - } -} -#[derive(Debug, PartialEq, Eq)] -struct CollectionPlanningFunction { - name: &'static str, - function: PlanningFunction, - signature: Signature, -} -// `function` is determined by `name` at registration, so hashing the name and -// signature agrees with the derived `Eq`. -impl Hash for CollectionPlanningFunction { - fn hash(&self, state: &mut H) { - self.name.hash(state); - self.signature.hash(state); - } -} -impl CollectionPlanningFunction { - fn output( - &self, - args: &[DataType], - nullable: &[bool], - literals: Option<&[Option<&DfScalarValue>]>, - ) -> Result<(DataType, bool)> { - let inputs = args - .iter() - .zip(nullable) - .map(|(dtype, null)| { - arrow_to_dtype(dtype) - .map(|dtype| (dtype, *null)) - .map_err(|e| DataFusionError::Plan(e.to_string())) - }) - .collect::>>()?; - let (dtype, nullable) = if matches!( - self.function, - PlanningFunction::Element | PlanningFunction::StructField - ) { - // DataFusion asks for argument-dependent types before canonical - // expression binding. Reuse the shared resolver over typed argument - // slots; final canonical binding also validates literal selectors. - let schema = Schema::new( - inputs - .into_iter() - .enumerate() - .map(|(index, (dtype, nullable))| { - Field::plain(format!("argument_{index}"), dtype, nullable) - }) - .collect(), - ); - let args = (0..schema.fields.len()) - .map(|index| { - if let Some(Some(value)) = literals.and_then(|args| args.get(index)) { - scalar_value_to_asap(value) - .map(QueryExpr::Literal) - .map_err(|error| DataFusionError::Plan(error.to_string())) - } else { - Ok(QueryExpr::Column(index)) - } - }) - .collect::>>()?; - match self.function { - PlanningFunction::Element => element_access_type(&args, &schema), - PlanningFunction::StructField => struct_field_type(&args, &schema), - PlanningFunction::Map(_) => unreachable!(), - } - } else if let PlanningFunction::Map(function) = self.function { - function.output_type(&inputs) - } else { - unreachable!() - } - .map_err(DataFusionError::Plan)?; - Ok((dtype_to_arrow(&dtype), nullable)) - } -} -impl ScalarUDFImpl for CollectionPlanningFunction { - fn name(&self) -> &str { - self.name - } - fn signature(&self) -> &Signature { - &self.signature - } - fn return_type(&self, args: &[DataType]) -> Result { - self.output( - args, - &args - .iter() - .map(|dtype| *dtype == DataType::Null) - .collect::>(), - None, - ) - .map(|output| output.0) - } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let types = args - .arg_fields - .iter() - .map(|field| field.data_type().clone()) - .collect::>(); - let nullable = args - .arg_fields - .iter() - .map(|field| field.is_nullable()) - .collect::>(); - let (dtype, nullable) = self.output(&types, &nullable, Some(args.scalar_arguments))?; - Ok(Arc::new(ArrowField::new(self.name, dtype, nullable))) - } - fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { - Err(DataFusionError::NotImplemented("collection planning adapter cannot execute; use a capable query engine or external exact sub-DAG".into())) - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn planning_adapter_explicitly_refuses_physical_execution() { - let adapter = CollectionPlanningFunction { - name: MAP_PLANNING_NAME, - function: PlanningFunction::Map(MapScalarFunction::Construct), - signature: Signature::any(0, Volatility::Immutable), - }; - let args = ScalarFunctionArgs { - args: vec![], - arg_fields: vec![], - number_rows: 1, - return_field: Arc::new(ArrowField::new("map", DataType::Null, true)), - config_options: Default::default(), - }; - assert!(matches!( - adapter.invoke_with_args(args), - Err(DataFusionError::NotImplemented(_)) - )); - let result = adapter.return_type(&[]).unwrap(); - let (expected, _) = MapScalarFunction::Construct.output_type(&[]).unwrap(); - assert_eq!(result, dtype_to_arrow(&expected)); - } -} diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs deleted file mode 100644 index 32ba893df..000000000 --- a/crates/frontend-sql/src/sql/expr.rs +++ /dev/null @@ -1,328 +0,0 @@ -use std::rc::Rc; - -use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; - -use asap_types::pre_asap::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; - -use crate::error::SqlError as LoweringError; - -use super::types::{arrow_to_dtype, scalar_value_to_asap}; -use super::Unresolved; - -pub(super) fn split_conjuncts(expr: &Expr) -> Vec<&Expr> { - match expr { - Expr::BinaryExpr(BinaryExpr { - left, - op: Operator::And, - right, - }) => { - let mut v = split_conjuncts(left); - v.extend(split_conjuncts(right)); - v - } - _ => vec![expr], - } -} - -/// Translate a DataFusion `Expr` to the canonical, unresolved DAG. -/// Returns `UnsupportedFeature` for anything not needed in v1. -pub(super) fn df_expr_to_unresolved(expr: &Expr) -> Result { - match expr { - // Preserve DataFusion's relation qualifier so a column name shared - // across a join (`a.k` vs `b.k`) resolves to the correct side. - Expr::Column(col) => Ok(Unresolved::Column(match &col.relation { - Some(rel) => ColumnRef::Qualified { - table: rel.to_string(), - name: col.name.clone(), - }, - None => ColumnRef::Named(col.name.clone()), - })), - - // Keep Arrow date literals equivalent to SQL CAST('YYYY-MM-DD' AS DATE), - // including typed nulls, without adding another canonical scalar variant. - Expr::Literal( - sv @ (datafusion::common::ScalarValue::Date32(_) - | datafusion::common::ScalarValue::Date64(_)), - _, - ) => { - let text = sv.cast_to(&datafusion::arrow::datatypes::DataType::Utf8)?; - // Arrow formats Date64 with a time suffix; the canonical Date has - // no time-of-day, just like Date64 catalog registration as Date32. - let text = match text { - datafusion::common::ScalarValue::Utf8(Some(value)) => { - ScalarValue::Utf8(value.split('T').next().unwrap().to_owned()) - } - other => scalar_value_to_asap(&other)?, - }; - Ok(Unresolved::Cast { - expr: Rc::new(Unresolved::Literal(text)), - to: asap_types::pre_asap::schema::DataType::Date, - try_cast: false, - }) - } - Expr::Literal(sv, _) => scalar_value_to_asap(sv).map(Unresolved::Literal), - - Expr::Alias(a) => df_expr_to_unresolved(&a.expr), - - Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op { - Operator::And => { - let parts = split_conjuncts(expr); - let lowered: Result, _> = - parts.iter().map(|e| df_expr_to_unresolved(e)).collect(); - Ok(Unresolved::BoolAnd(lowered?)) - } - Operator::Or => { - let parts = split_disjuncts(expr); - let lowered: Result, _> = - parts.iter().map(|e| df_expr_to_unresolved(e)).collect(); - Ok(Unresolved::BoolOr(lowered?)) - } - Operator::Eq => compare(left, CompareOpKind::Eq, right), - Operator::NotEq => compare(left, CompareOpKind::Ne, right), - Operator::Lt => compare(left, CompareOpKind::Lt, right), - Operator::LtEq => compare(left, CompareOpKind::Le, right), - Operator::Gt => compare(left, CompareOpKind::Gt, right), - Operator::GtEq => compare(left, CompareOpKind::Ge, right), - // BinaryExpr LIKE/ILIKE operators (from optimizer rewrites) - Operator::LikeMatch => compare(left, CompareOpKind::Like, right), - Operator::ILikeMatch => compare(left, CompareOpKind::ILike, right), - Operator::NotLikeMatch => compare(left, CompareOpKind::NotLike, right), - Operator::NotILikeMatch => compare(left, CompareOpKind::NotILike, right), - // Arithmetic - Operator::Plus => arith(left, ArithmeticOpKind::Add, right), - Operator::Minus => arith(left, ArithmeticOpKind::Sub, right), - Operator::Multiply => arith(left, ArithmeticOpKind::Mul, right), - Operator::Divide => arith(left, ArithmeticOpKind::Div, right), - Operator::Modulo => arith(left, ArithmeticOpKind::Mod, right), - other => Err(LoweringError::UnsupportedFeature(format!( - "operator: {other:?}" - ))), - }, - - // SQL LIKE / ILIKE (dedicated expr node from the SQL parser) - Expr::Like(like) => { - let op = match (like.negated, like.case_insensitive) { - (false, false) => CompareOpKind::Like, - (true, false) => CompareOpKind::NotLike, - (false, true) => CompareOpKind::ILike, - (true, true) => CompareOpKind::NotILike, - }; - compare(&like.expr, op, &like.pattern) - } - - // Unary minus: negate literals directly; wrap others in -1 * x. - Expr::Negative(inner) => { - let inner = df_expr_to_unresolved(inner)?; - match inner { - Unresolved::Literal(ScalarValue::Int64(v)) => { - Ok(Unresolved::Literal(ScalarValue::Int64(-v))) - } - Unresolved::Literal(ScalarValue::Float64(v)) => { - Ok(Unresolved::Literal(ScalarValue::Float64(-v))) - } - other => Ok(Unresolved::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Rc::new(Unresolved::Literal(ScalarValue::Int64(-1))), - right: Rc::new(other), - }), - } - } - - // SQL CASE expression - Expr::Case(c) => { - let operand = c - .expr - .as_ref() - .map(|e| df_expr_to_unresolved(e).map(Rc::new)) - .transpose()?; - let branches = c - .when_then_expr - .iter() - .map(|(when, then)| { - Ok((df_expr_to_unresolved(when)?, df_expr_to_unresolved(then)?)) - }) - .collect::, LoweringError>>()?; - let else_expr = c - .else_expr - .as_ref() - .map(|e| df_expr_to_unresolved(e).map(Rc::new)) - .transpose()?; - Ok(Unresolved::Case { - operand, - branches, - else_expr, - }) - } - - Expr::Not(inner) => Ok(Unresolved::Not(Rc::new(df_expr_to_unresolved(inner)?))), - - Expr::IsNull(inner) => Ok(Unresolved::IsNull(Rc::new(df_expr_to_unresolved(inner)?))), - - Expr::IsNotNull(inner) => Ok(Unresolved::IsNotNull(Rc::new(df_expr_to_unresolved( - inner, - )?))), - - Expr::Cast(c) => { - let inner = df_expr_to_unresolved(&c.expr)?; - let to = arrow_to_dtype(c.field.data_type())?; - Ok(Unresolved::Cast { - expr: Rc::new(inner), - to, - try_cast: false, - }) - } - - // TRY_CAST returns NULL on conversion failure; preserve that semantic. - Expr::TryCast(c) => { - let inner = df_expr_to_unresolved(&c.expr)?; - let to = arrow_to_dtype(c.field.data_type())?; - Ok(Unresolved::Cast { - expr: Rc::new(inner), - to, - try_cast: true, - }) - } - - Expr::InList(il) => { - let expr = df_expr_to_unresolved(&il.expr)?; - let list: Result, _> = il.list.iter().map(df_expr_to_unresolved).collect(); - Ok(Unresolved::InList { - expr: Rc::new(expr), - list: list?, - negated: il.negated, - }) - } - - Expr::Between(b) => { - // Normalize: `x BETWEEN low AND high` → `x >= low AND x <= high`. - // `x NOT BETWEEN low AND high` → `x < low OR x > high`. - let x_low = compare(&b.expr, CompareOpKind::Ge, &b.low)?; - let x_high = compare(&b.expr, CompareOpKind::Le, &b.high)?; - if b.negated { - // NOT BETWEEN: invert each side - let lt = compare(&b.expr, CompareOpKind::Lt, &b.low)?; - let gt = compare(&b.expr, CompareOpKind::Gt, &b.high)?; - Ok(Unresolved::BoolOr(vec![lt, gt])) - } else { - Ok(Unresolved::BoolAnd(vec![x_low, x_high])) - } - } - - // `NOW()` / `CURRENT_TIMESTAMP` read the SQL statement evaluation - // time. Keep this timestamp-typed leaf distinct from PromQL's - // Float64 Unix-seconds `EvalTimestamp`. Issue #184. - Expr::ScalarFunction(sf) - if sf.args.is_empty() - && matches!( - sf.func.name().to_ascii_lowercase().as_str(), - "now" | "current_timestamp" - ) => - { - Ok(Unresolved::CurrentTimestamp) - } - - Expr::ScalarFunction(sf) => { - let args: Result, _> = sf.args.iter().map(df_expr_to_unresolved).collect(); - Ok(Unresolved::FunctionCall { - name: if sf.func.name().eq_ignore_ascii_case("arrayelement") { - "asap_element_access".into() - } else if sf.func.name().eq_ignore_ascii_case("tupleelement") { - "asap_struct_field".into() - } else if sf.func.name() == super::collection_planning::MAP_PLANNING_NAME { - "map".into() - } else { - sf.func.name().to_string() - }, - args: args?, - }) - } - - // Subquery-valued expressions in a predicate/projection — `x > (SELECT - // …)`, `x IN (SELECT …)`, `EXISTS (SELECT …)`. These need a subquery - // node in the unresolved expression IR (and a correlated-vs-uncorrelated - // decision); rejected cleanly until that lands rather than mislowered. - // Derived tables in `FROM` (the common nesting shape) ARE supported — - // see `lower_plan`'s `SubqueryAlias` arm. - Expr::ScalarSubquery(_) | Expr::InSubquery(_) | Expr::Exists(_) => Err( - LoweringError::UnsupportedFeature("subquery-valued expression in predicate".into()), - ), - - other => Err(LoweringError::UnsupportedFeature(format!( - "expression: {}", - other - ))), - } -} - -pub(super) fn compare( - left: &Expr, - op: CompareOpKind, - right: &Expr, -) -> Result { - Ok(Unresolved::Compare { - left: Rc::new(df_expr_to_unresolved(left)?), - op, - right: Rc::new(df_expr_to_unresolved(right)?), - }) -} - -pub(super) fn arith( - left: &Expr, - op: ArithmeticOpKind, - right: &Expr, -) -> Result { - Ok(Unresolved::Arithmetic { - op, - left: Rc::new(df_expr_to_unresolved(left)?), - right: Rc::new(df_expr_to_unresolved(right)?), - }) -} - -pub(super) fn split_disjuncts(expr: &Expr) -> Vec<&Expr> { - match expr { - Expr::BinaryExpr(BinaryExpr { - left, - op: Operator::Or, - right, - }) => { - let mut v = split_disjuncts(left); - v.extend(split_disjuncts(right)); - v - } - _ => vec![expr], - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::pre_asap::schema::DataType; - use datafusion::common::ScalarValue as DfScalarValue; - - // Typed Arrow dates normalize to the same typed form as SQL date casts. - #[test] - fn arrow_date_literals_preserve_value_and_type() { - for (value, expected) in [ - ( - DfScalarValue::Date32(Some(0)), - ScalarValue::Utf8("1970-01-01".into()), - ), - ( - DfScalarValue::Date64(Some(-86_400_000)), - ScalarValue::Utf8("1969-12-31".into()), - ), - (DfScalarValue::Date32(None), ScalarValue::Null), - (DfScalarValue::Date64(None), ScalarValue::Null), - ] { - let actual = df_expr_to_unresolved(&Expr::Literal(value, None)).unwrap(); - assert_eq!( - actual, - Unresolved::Cast { - expr: Rc::new(Unresolved::Literal(expected)), - to: DataType::Date, - try_cast: false, - } - ); - } - } -} diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs deleted file mode 100644 index 3c708a2ed..000000000 --- a/crates/frontend-sql/src/sql/mod.rs +++ /dev/null @@ -1,2523 +0,0 @@ -//! SQL → the canonical, unresolved -//! [`UnresolvedQueryExpr`](asap_types::pre_asap::query_expr::UnresolvedQueryExpr) -//! (`QueryExpr`). -//! -//! Parses SQL via DataFusion (over the catalog's registered tables), then -//! walks the unoptimized `LogicalPlan` and emits `UnresolvedQueryExpr` nodes with -//! unresolved `ColumnRef`s directly (issue #179) — the same DAG shape -//! [`resolve_root`](asap_types::pre_asap::resolve_root) binds to canonical, -//! positional `QueryExpr`. Unlike PromQL's front end, SQL's -//! Ordinary SQL `Aggregate` nodes are `Reduction::Reduce`. The explicit -//! `asap_rate`/`asap_increase` bridge is the narrow exception: it -//! spells a time-series range reducer with an explicit value, time-index, and -//! window and therefore lowers to the same `TimeRange` + `PerEntity` shape as -//! its PromQL counterpart. The front end also has to fold a `WHERE` directly -//! over a bare table scan onto -//! `Scan.predicates` itself (`filter_or_fold`) — canonical's invariant that a -//! `Filter` never sits directly over a `Scan` — since front ends producing -//! this shape are responsible for it now, not a converter. -//! -//! Heavy-hitter `topk` recognition (`ORDER BY count(...) DESC LIMIT k`) is -//! *not* done here: SQL emits a plain `Sort`/`Limit`, and the shared -//! `canonicalize` pass (issue #34, run by `resolve_root`) recognises the -//! count-ranked shape positionally, so a SQL `ORDER BY`/`LIMIT` and a PromQL -//! `topk(...)` converge without either front end special-casing the other's -//! syntax. - -use std::rc::Rc; -use std::sync::Arc; -use std::time::Duration; - -use datafusion::arrow::compute::kernels::cast_utils::parse_interval_month_day_nano; -use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, FieldRef}; -use datafusion::catalog::MemorySchemaProvider; -use datafusion::common::config::ConfigOptions; -use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{Column as DfColumn, DFSchema, ScalarValue as DfScalarValue}; -use datafusion::datasource::MemTable; -use datafusion::functions_aggregate::count::count_udaf; -use datafusion::functions_aggregate::sum::sum_udaf; -use datafusion::logical_expr::expr::AggregateFunction; -use datafusion::logical_expr::expr_rewriter::FunctionRewrite; -use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs}; -use datafusion::logical_expr::{ - self, lit, AggregateUDF, Case, ColumnarValue, Distinct, Expr, ExprSchemable, JoinType, - LogicalPlan, PartitionEvaluator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, - SimpleAggregateUDF, TypeSignature, Volatility, WindowFrameBound as DfWindowFrameBound, - WindowFrameUnits as DfWindowFrameUnits, WindowFunctionDefinition, WindowUDF, WindowUDFImpl, -}; -use datafusion::optimizer::analyzer::function_rewrite::ApplyFunctionRewrites; -use datafusion::optimizer::{AnalyzerRule, OptimizerConfig}; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion::sql::parser::DFParser; -use datafusion::sql::sqlparser::dialect::GenericDialect; - -use asap_sql_function_catalog::{AggSemantic, Arity, RewriteKind}; -use asap_types::pre_asap::agg_intent::AggIntent; -use asap_types::pre_asap::query_expr::{ - GroupKeys, Predicate, ProjectItem, Reduction, SortKey, Source, - UnresolvedQueryExpr as Unresolved, WindowFrame, WindowFrameBound, WindowFrameOffset, - WindowFrameUnits, -}; -use asap_types::pre_asap::schema::{DataType, FieldDataType, Schema}; -use asap_types::pre_asap::{ - resolve_column_ref, resolve_root, ColumnRef, CompareOpKind, JoinKind, RelationalSetOpKind, - ScalarValue, WindowFuncKind, -}; -use asap_types::types::AccuracyTarget; -use asap_types::workload::SqlDialect; - -use crate::error::SqlError as LoweringError; - -mod clickhouse_ast; -mod collection_planning; -mod expr; -mod types; - -pub use types::SqlCatalog; - -use self::expr::df_expr_to_unresolved; -use self::types::{arrow_to_dtype, scalar_value_to_asap, schema_to_arrow}; - -std::thread_local! { - static ACCURACY: std::cell::RefCell = - const { std::cell::RefCell::new(AccuracyTarget::Exact) }; -} - -/// RAII guard installing `accuracy` as the ambient accuracy target for the -/// current thread's lowering, restoring the prior value on drop — same -/// ambient-thread-local shape as `asap_frontend_promql::promql`'s -/// `AccuracyGuard`, for the same reason: it injects `accuracy` into the deep -/// `lower_plan` recursion without a parameter on every one of its -/// signatures, consulted only at the couple of sites that build an -/// accuracy-bearing `AggIntent`. -struct AccuracyGuard(AccuracyTarget); - -impl AccuracyGuard { - fn install(accuracy: AccuracyTarget) -> Self { - let prev = ACCURACY.with(|a| a.replace(accuracy)); - AccuracyGuard(prev) - } -} - -impl Drop for AccuracyGuard { - fn drop(&mut self) { - ACCURACY.with(|a| *a.borrow_mut() = std::mem::replace(&mut self.0, AccuracyTarget::Exact)); - } -} - -fn current_accuracy() -> AccuracyTarget { - ACCURACY.with(|a| a.borrow().clone()) -} - -/// Lowers SQL strings to the canonical [`UnresolvedQueryExpr`](asap_types::pre_asap::UnresolvedQueryExpr) -/// over a table [`SqlCatalog`]. Call -/// [`resolve_root`](asap_types::pre_asap::resolve_root) on the result for -/// the canonical, resolved DAG. -pub struct SqlLowerer<'a> { - catalog: &'a SqlCatalog, - dialect: SqlDialect, -} - -impl<'a> SqlLowerer<'a> { - pub fn new(catalog: &'a SqlCatalog) -> Self { - Self { - catalog, - dialect: SqlDialect::DataFusionSQL, - } - } - - /// Parse under a specific SQL dialect (e.g. `ClickhouseSQL`, which maps to - /// sqlparser's vendored `ClickHouseDialect` — array-lambda syntax and - /// `arr[-1]` indexing parse under it that don't parse generically). This - /// only changes *parsing*: a ClickHouse-only builtin function not listed - /// in `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (`uniqExact` and - /// `countIf` are; most of ClickHouse's builtin surface isn't yet) is - /// still unknown to DataFusion's planner and still fails there, and - /// `ElasticSQL` has no vendored parser at all. - pub fn with_dialect(catalog: &'a SqlCatalog, dialect: SqlDialect) -> Self { - Self { catalog, dialect } - } - - /// Parse + lower a SQL query to the canonical, unresolved shape, threading - /// `accuracy` onto every approximate intent (`Count`, `Quantile`, - /// `Cardinality`) as it is built. - /// - /// The `AccuracyGuard` installs *after* the only `.await` point - /// (DataFusion statement planning) — `lower_plan` itself is synchronous, so once it starts - /// there is no further suspension point that could move this task to a - /// different OS thread out from under a thread-local set beforehand. - /// - /// Runs `ApplyFunctionRewrites` — the single `AnalyzerRule` DataFusion's - /// own `Analyzer` uses internally to apply `FunctionRewrite`s, called - /// directly rather than through `Analyzer::execute_and_check` — over the - /// raw parsed plan before lowering, carrying only - /// `ClickHouseBuiltinRewrite` (catalog-driven, see its own doc — it - /// covers every `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` entry, - /// not just one). `ctx.sql(...).into_unoptimized_plan()` alone returns - /// `SqlToRel`'s output untouched, and a `FunctionRewrite` only ever runs - /// as part of this rule, so calling it directly is unavoidable to make - /// the rewrite fire. Its `analyze()` already does a full - /// `transform_up_with_subqueries` over the whole plan, so it needs no - /// wrapping `Analyzer` at all — deliberately not - /// `Analyzer::execute_and_check` (whether with the default 5-rule - /// analyzer or an empty one carrying just this rewrite): that method - /// runs an unconditional post-check (`check_plan`, hardcoded, not itself - /// a rule) that isn't wanted here — e.g. it independently rejects a - /// multi-column `IN (subquery)` before `lower_in_subquery`'s own arity - /// check would. Going straight to `ApplyFunctionRewrites` avoids that - /// entirely: zero behavior change for every query that doesn't call a - /// catalog-listed ClickHouse builtin. - pub async fn lower( - &self, - sql: &str, - accuracy: &AccuracyTarget, - ) -> Result { - let ctx = self.build_context()?; - let state = ctx.state(); - let statement = if matches!(self.dialect, SqlDialect::ClickhouseSQL) { - let mut statement = - state.sql_to_statement(sql, &datafusion::config::Dialect::ClickHouse)?; - if let datafusion::sql::parser::Statement::Statement(ast) = &mut statement { - clickhouse_ast::normalize(ast); - } - statement - } else { - let mut statements = DFParser::parse_sql_with_dialect(sql, &GenericDialect)?; - let (Some(statement), true) = (statements.pop_front(), statements.is_empty()) else { - return Err(LoweringError::UnsupportedFeature( - "exactly one SQL statement per query".into(), - )); - }; - statement - }; - let plan = state.statement_to_plan(statement).await?; - let rewriter = ApplyFunctionRewrites::new(vec![Arc::new(ClickHouseBuiltinRewrite)]); - let plan = rewriter.analyze(plan, &ctx.state().options())?; - // Output schemas omit predicate and nested-expression types. Check the - // typed SQL plan before lowering erases fixed-duration units. - plan.apply_with_subqueries(|node| { - let mut schema = DFSchema::empty(); - for input in node.inputs() { - schema.merge(input.schema()); - } - schema.merge(node.schema()); - node.apply_expressions(|expr| { - expr.apply(|nested| { - if let Expr::BinaryExpr(binary) = nested { - if binary.op == logical_expr::Operator::Minus - // DataFusion types `date - date` as an Int64 day - // count, which the canonical DAG has no shape for. - && (matches!( - (binary.left.get_type(&schema)?, binary.right.get_type(&schema)?), - (ArrowDataType::Date32, ArrowDataType::Date32) - | (ArrowDataType::Date64, ArrowDataType::Date64) - ) || matches!( - nested.get_type(&schema)?, - ArrowDataType::Duration(_) - )) - { - return Err(datafusion::common::DataFusionError::Plan( - "temporal subtraction produces an unsupported duration type".into(), - )); - } - } - Ok(TreeNodeRecursion::Continue) - }) - }) - })?; - let _guard = AccuracyGuard::install(accuracy.clone()); - self.lower_plan(&plan) - } - - /// Register the catalog tables (empty Arrow `MemTable`s) so DataFusion can - /// resolve table/column references during planning. - fn build_context(&self) -> Result { - let dialect_name = match &self.dialect { - SqlDialect::DataFusionSQL => "generic", - SqlDialect::ClickhouseSQL => "ClickHouse", - SqlDialect::ElasticSQL => { - return Err(LoweringError::UnsupportedDialect("ElasticSQL".into())) - } - }; - let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", dialect_name); - let ctx = SessionContext::new_with_config(config); - if matches!(self.dialect, SqlDialect::ClickhouseSQL) { - collection_planning::register(&ctx); - } - // A catalog key like "bgp.bgp_updates" schema-qualifies the table - // (e.g. a ClickHouse database name). DataFusion requires the parent - // schema to be registered before a qualified table can be, so create - // it on demand. - let catalog_provider = ctx.catalog("datafusion").ok_or_else(|| { - LoweringError::InvalidExpression("default \"datafusion\" catalog missing".into()) - })?; - for (name, schema) in &self.catalog.tables { - if let Some((schema_name, _)) = name.split_once('.') { - if catalog_provider.schema(schema_name).is_none() { - catalog_provider - .register_schema(schema_name, Arc::new(MemorySchemaProvider::new()))?; - } - } - let arrow_schema = Arc::new(schema_to_arrow(schema)); - let mem_table = MemTable::try_new(arrow_schema, vec![vec![]])?; - ctx.register_table(name.as_str(), Arc::new(mem_table))?; - } - // Register a stub `AggregateUDF` for every catalog-listed - // ClickHouse-only builtin, purely so DataFusion's planner can - // resolve its name during parsing — `lower()` rewrites every call - // site to a native DataFusion aggregate via `ClickHouseBuiltinRewrite` - // before `lower_plan` sees it. - for builtin in asap_sql_function_catalog::CLICKHOUSE_BUILTINS { - ctx.register_udaf(clickhouse_builtin_stub_udaf(builtin.name, builtin.arity)); - } - // Register a stub `ScalarUDF` for every catalog-listed ClickHouse-only - // *scalar* builtin — same reason as the `AggregateUDF` loop above - // (DataFusion otherwise rejects the call as an unknown function - // during `SqlToRel` conversion), but with no rewrite step to follow: - // `df_expr_to_unresolved`'s `Expr::ScalarFunction` arm already lowers - // any scalar call generically to `Unresolved::FunctionCall { name, - // args }`, so registering the stub is the entire fix (issue #230). - for builtin in asap_sql_function_catalog::CLICKHOUSE_SCALAR_BUILTINS { - ctx.register_udf(clickhouse_scalar_builtin_stub_udf( - builtin.name, - builtin.arity, - )); - } - // Planning-only relation markers. They let a workload author state - // the PromQL temporal/classic-histogram semantics of an equivalent SQL - // rewrite without teaching the canonical IR a second, SQL-specific - // spelling of either operation. `lower_projection` consumes these - // calls; they can never survive as executable scalar functions. - for (name, arity) in [ - ("asap_promql_subquery", Arity::Exact(2)), - ("asap_histogram_quantile", Arity::Exact(1)), - ] { - ctx.register_udf(clickhouse_scalar_builtin_stub_udf(name, arity)); - } - // Register a stub `WindowUDF` for every catalog-listed ClickHouse-only - // *window* builtin — same reason as the two loops above, but with no - // rewrite step to follow: `lower_window_func_kind` already maps each - // name directly to its own `WindowFuncKind` variant (issue #267). - for builtin in asap_sql_function_catalog::CLICKHOUSE_WINDOW_BUILTINS { - ctx.register_udwf(clickhouse_window_builtin_stub_udwf( - builtin.name, - builtin.arity, - )); - } - Ok(ctx) - } - - fn lower_plan(&self, plan: &LogicalPlan) -> Result { - match plan { - LogicalPlan::TableScan(scan) => self.lower_table_scan(scan), - LogicalPlan::Filter(filter) => self.lower_filter(filter), - LogicalPlan::Projection(proj) => self.lower_projection(proj), - LogicalPlan::Aggregate(agg) => self.lower_aggregate(agg), - LogicalPlan::Sort(sort) => self.lower_sort(sort), - LogicalPlan::Limit(limit) => self.lower_limit(limit), - LogicalPlan::Distinct(d) => match d { - Distinct::On(_) => Err(LoweringError::UnsupportedFeature("DISTINCT ON".into())), - Distinct::All(input) => Ok(Unresolved::Dedup { - cols: vec![], - child: Rc::new(self.lower_plan(input)?), - }), - }, - LogicalPlan::Union(u) => { - // Fold n inputs left-associatively into SetOp { Union, all: true }. - let mut iter = u.inputs.iter(); - let first = iter - .next() - .ok_or_else(|| LoweringError::InvalidExpression("empty union".into()))?; - let first_expr = self.lower_plan(first)?; - iter.try_fold(first_expr, |left, right_plan| { - Ok(Unresolved::SetOp { - kind: RelationalSetOpKind::Union, - all: true, - left: Rc::new(left), - right: Rc::new(self.lower_plan(right_plan)?), - }) - }) - } - LogicalPlan::Window(window) => self.lower_window(window), - LogicalPlan::Join(join) => self.lower_join(join), - LogicalPlan::Subquery(_) => Err(LoweringError::UnsupportedFeature("subquery".into())), - LogicalPlan::SubqueryAlias(alias) => { - // An alias over a table re-qualifies the scan's columns with the - // alias (so `a.col` / `b.col` in a self-join disambiguate). - match alias.input.as_ref() { - LogicalPlan::TableScan(scan) => { - self.scan_source(&scan.table_name.to_string(), &alias.alias.to_string()) - } - // A *derived table* / inline view — `FROM (SELECT …) t`, the - // SQL counterpart of PromQL function nesting (an aggregate - // over an aggregate, a filter over a derived aggregate, …). - // Lower the inner plan, then re-qualify its output columns - // with the alias so `t.col` resolves to *this* relation — and, - // critically, so a join over two derived tables disambiguates - // its keys instead of both binding to the first bare-name - // match (issue #66). The inner column *names* are unchanged; - // only the qualifier is stamped. - other => { - let alias_name = alias.alias.to_string(); - match self.lower_plan(other)? { - // The derived SELECT list already lowered to a - // Projection — stamp the alias onto it, no extra node. - Unresolved::Project { cols, child, .. } => Ok(Unresolved::Project { - cols, - qualifier: Some(alias_name), - child, - }), - // Otherwise (e.g. a `LIMIT` or `DISTINCT` sub-plan) wrap - // in an identity projection that re-qualifies each - // output column. Names come from the sub-plan's schema. - inner => { - let cols = alias - .input - .schema() - .fields() - .iter() - .map(|f| ProjectItem { - alias: Some(f.name().clone()), - expr: Unresolved::Column(ColumnRef::Named( - f.name().clone(), - )), - }) - .collect(); - Ok(Unresolved::Project { - cols, - qualifier: Some(alias_name), - child: Rc::new(inner), - }) - } - } - } - } - } - other => Err(LoweringError::UnsupportedFeature(format!( - "plan node: {}", - other.display() - ))), - } - } - - /// `WHERE` — a conjunction of ordinary predicates plus, possibly, subquery - /// predicates (issue #111). - /// - /// `c IN (SELECT …)` and `EXISTS (…)` are not expressions over rows; they are - /// *joins*. Each such conjunct peels off into a semi- / anti-join above the - /// filter's input, and the remaining conjuncts stay as an ordinary `Filter`. - /// - /// The residual filter is applied **below** the joins, which is where it sat - /// before: a semi-join only ever drops left rows, so the two orders agree — - /// and keeping the fold-onto-`Scan` (`filter_or_fold`) below the joins - /// matches where the old converter folded it too. - fn lower_filter(&self, filter: &logical_expr::Filter) -> Result { - let mut conjuncts = Vec::new(); - split_conjunction(&filter.predicate, &mut conjuncts); - let (subqueries, residual): (Vec<_>, Vec<_>) = conjuncts - .into_iter() - .partition(|e| matches!(e, Expr::InSubquery(_) | Expr::Exists(_))); - - let input = self.lower_plan(&filter.input)?; - let mut node = match rebuild_conjunction(&residual) { - Some(pred) => filter_or_fold(df_expr_to_unresolved(&pred)?, input), - None => input, - }; - for sq in subqueries { - node = match sq { - Expr::InSubquery(is) => self.lower_in_subquery(is, node)?, - Expr::Exists(ex) => self.lower_exists(ex, node)?, - _ => unreachable!("partitioned above"), - }; - } - Ok(node) - } - - /// `c IN (SELECT k FROM …)` → a semi-join on `c = k` (issue #111). - fn lower_in_subquery( - &self, - is: &logical_expr::expr::InSubquery, - left: Unresolved, - ) -> Result { - if is.negated { - // `NOT IN` is not an anti-join. Under three-valued logic a single - // NULL among the subquery's rows makes `c NOT IN (…)` UNKNOWN for - // every `c`, so the query returns nothing — while an anti-join - // returns every unmatched left row. Reject rather than mislower. - return Err(LoweringError::UnsupportedFeature( - "NOT IN (subquery): its NULL semantics are not an anti-join".into(), - )); - } - if !is.subquery.outer_ref_columns.is_empty() { - return Err(LoweringError::UnsupportedFeature( - "correlated IN (subquery)".into(), - )); - } - let inner = is.subquery.subquery.as_ref(); - let fields = inner.schema().fields(); - if fields.len() != 1 { - return Err(LoweringError::InvalidExpression(format!( - "IN (subquery) must select exactly one column, got {}", - fields.len() - ))); - } - let key = &fields[0]; - // Project the key under a name the outer relation cannot also carry. The - // join predicate resolves against the concatenated `left ++ right` - // schema, and a bare `hosts.service` over an unqualified subquery output - // falls back to a name lookup that finds the *left's* `service` first — - // silently making the predicate `service = service`, i.e. always true. - let right = match inner { - // Rebuild the subquery's projection with the synthetic alias, so a - // computed key (`SELECT bytes + 1 …`) is named rather than becoming - // the anonymous `col_0` that nothing can reference. - LogicalPlan::Projection(p) if p.expr.len() == 1 => Unresolved::Project { - cols: vec![ProjectItem { - alias: Some(IN_SUBQUERY_KEY.to_string()), - expr: df_expr_to_unresolved(unalias(&p.expr[0]))?, - }], - qualifier: None, - child: Rc::new(self.lower_plan(&p.input)?), - }, - other => Unresolved::Project { - cols: vec![ProjectItem { - alias: Some(IN_SUBQUERY_KEY.to_string()), - expr: Unresolved::Column(ColumnRef::Named(key.name().clone())), - }], - qualifier: None, - child: Rc::new(self.lower_plan(other)?), - }, - }; - Ok(Unresolved::Join { - kind: JoinKind::Semi, - pred: Predicate(Rc::new(Unresolved::Compare { - left: Rc::new(df_expr_to_unresolved(&is.expr)?), - op: CompareOpKind::Eq, - right: Rc::new(Unresolved::Column(ColumnRef::Named( - IN_SUBQUERY_KEY.to_string(), - ))), - })), - left: Rc::new(left), - right: Rc::new(right), - }) - } - - /// `[NOT] EXISTS (SELECT … WHERE inner.k = outer.k)` → a semi- / anti-join - /// on the correlation predicate (issue #111). - fn lower_exists( - &self, - ex: &logical_expr::expr::Exists, - left: Unresolved, - ) -> Result { - let kind = if ex.negated { - JoinKind::Anti - } else { - JoinKind::Semi - }; - // A semi-join discards the right side's columns, and `SELECT 1` projects - // the correlation columns away — so drop the subquery's projections and - // join against what they sit on. - let mut inner = ex.subquery.subquery.as_ref(); - while let LogicalPlan::Projection(p) = inner { - inner = &p.input; - } - // Lift the correlated conjuncts out of the subquery's filter; they are - // the join predicate. Whatever is left stays an ordinary inner filter. - let (inner, correlation) = split_correlation(inner)?; - let right = self.lower_plan(&inner)?; - // No correlation conjunct (a genuinely uncorrelated `EXISTS`) means - // the join condition is unconditionally true — same convention as an - // unconditional `JOIN` (`lower_join`, below). - let pred = match correlation { - Some(e) => Predicate(Rc::new(df_expr_to_unresolved(&e)?)), - None => Predicate(Rc::new(Unresolved::Literal(ScalarValue::Boolean(true)))), - }; - Ok(Unresolved::Join { - kind, - pred, - left: Rc::new(left), - right: Rc::new(right), - }) - } - - /// Table leaf — carries the catalog's resolved schema directly on `Scan` - /// (`schema: Some(_)`), so `resolve_root`'s SchemaResolver doesn't need to - /// usage-derive it (SQL is never schemaless). Projection pushdown is left - /// to the enclosing `Project` (DataFusion's unoptimized plan sets no - /// projection). - fn lower_table_scan( - &self, - scan: &logical_expr::TableScan, - ) -> Result { - let table = scan.table_name.to_string(); - self.scan_source(&table, &table) - } - - /// A `Scan` over catalog table `table`, with its columns qualified by - /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so - /// `Qualified` column refs resolve to the right side across a join. - fn scan_source(&self, table: &str, qualifier: &str) -> Result { - let schema = self - .catalog - .tables - .get(table) - .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; - let qualified = Schema { - fields: schema - .fields - .iter() - .cloned() - .map(|c| c.with_table(qualifier)) - .collect(), - time_index: schema.time_index, - unique_keys: schema.unique_keys.clone(), - // Catalog-backed: the table's columns are fully declared → closed. - closed: true, - }; - Ok(Unresolved::Scan { - source: Source::Table { - table_ref: table.to_string(), - }, - predicates: vec![], - schema: Some(qualified), - }) - } - - /// ⋈ — equijoin. The `on` key pairs become `left = right` comparisons, - /// AND-ed with any non-equi `filter`, into the join predicate — still - /// name-based here (like a `WHERE`); `resolve_root` derives the - /// concatenated output schema downstream. Semi/anti/mark joins have no - /// canonical counterpart yet and are rejected. - fn lower_join(&self, join: &logical_expr::Join) -> Result { - let kind = match join.join_type { - JoinType::Inner => JoinKind::Inner, - JoinType::Left => JoinKind::Left, - JoinType::Right => JoinKind::Right, - JoinType::Full => JoinKind::Full, - other => { - return Err(LoweringError::UnsupportedFeature(format!( - "join type: {other:?}" - ))) - } - }; - let mut conjuncts = join - .on - .iter() - .map(|(l, r)| { - Ok(Unresolved::Compare { - left: Rc::new(df_expr_to_unresolved(l)?), - op: CompareOpKind::Eq, - right: Rc::new(df_expr_to_unresolved(r)?), - }) - }) - .collect::, LoweringError>>()?; - if let Some(filter) = &join.filter { - conjuncts.push(df_expr_to_unresolved(filter)?); - } - let pred = Predicate(Rc::new(match conjuncts.len() { - // No condition (a CROSS JOIN) is unconditionally true. - 0 => Unresolved::Literal(ScalarValue::Boolean(true)), - 1 => conjuncts.pop().unwrap(), - _ => Unresolved::BoolAnd(conjuncts), - })); - Ok(Unresolved::Join { - kind, - pred, - left: Rc::new(self.lower_plan(&join.left)?), - right: Rc::new(self.lower_plan(&join.right)?), - }) - } - - /// `func(args) OVER (PARTITION BY … ORDER BY … ROWS/RANGE BETWEEN …)`. One - /// window function per plan node. - fn lower_window(&self, window: &logical_expr::Window) -> Result { - if window.window_expr.len() > 1 { - return Err(LoweringError::UnsupportedFeature(format!( - "multiple window functions in one plan node (got {}); split them", - window.window_expr.len() - ))); - } - let child = Rc::new(self.lower_plan(&window.input)?); - let first = window - .window_expr - .first() - .ok_or_else(|| LoweringError::InvalidExpression("empty window expression".into()))?; - let Expr::WindowFunction(wf) = first else { - return Err(LoweringError::InvalidExpression( - "expected a window function in Window plan node".into(), - )); - }; - let func = lower_window_func_kind(&wf.fun)?; - let mut args = wf - .params - .args - .iter() - .map(df_expr_to_unresolved) - .collect::, _>>()?; - // Nth_value: lift N from the (literal) 2nd arg, keep only the column. - let func = if matches!(func, WindowFuncKind::NthValue(None)) { - let n = match args.get(1) { - Some(Unresolved::Literal(ScalarValue::Int64(n))) if *n > 0 => *n as u64, - other => { - return Err(LoweringError::InvalidExpression(format!( - "NTH_VALUE requires a positive integer literal 2nd arg, got {other:?}" - ))) - } - }; - args.truncate(1); - WindowFuncKind::NthValue(Some(n)) - } else { - func - }; - let partition_by = wf - .params - .partition_by - .iter() - .map(expr_to_group_ref) - .collect::, _>>()?; - let order_by = wf - .params - .order_by - .iter() - .map(|s| { - df_expr_to_unresolved(&s.expr).map(|expr| SortKey { - expr, - ascending: s.asc, - nulls_first: s.nulls_first, - }) - }) - .collect::, _>>()?; - let frame = lower_window_frame(&wf.params.window_frame)?; - // The window plan's schema is `[input fields …, window output]`; the last - // field is the window column's name (what an enclosing Project references). - let output_name = window - .schema - .fields() - .last() - .map(|f| f.name().clone()) - .unwrap_or_else(|| "window".into()); - Ok(Unresolved::SQLWindowFunc { - func, - args, - partition_by: partition_by.into(), - order_by, - frame: Some(frame), - output_name, - child, - }) - } - - fn lower_projection( - &self, - proj: &logical_expr::Projection, - ) -> Result { - if let Some(bridge) = planning_bridge(proj)? { - let input = self.lower_plan(&proj.input)?; - return Ok(match bridge { - PlanningBridge::PromqlSubquery { range, resolution } => { - let child = Rc::new(temporal_bridge_projection(proj, input)?); - Unresolved::PromqlSubquery { - range, - resolution: Some(resolution), - child, - } - } - PlanningBridge::HistogramQuantile { q } => Unresolved::Aggregate { - // The marker is the projection's only column: one histogram. - reduction: Reduction::Reduce(GroupKeys::none()), - measures: vec![AggIntent::HistogramQuantile { - q, - le: ColumnRef::Named("le".into()), - }], - output_names: vec!["value".into()], - filters: vec![], - having: None, - child: Rc::new(input), - }, - }); - } - let child = Rc::new(self.lower_plan(&proj.input)?); - let temporal_input = plan_has_temporal_aggregate(&proj.input); - let cols = proj - .expr - .iter() - .map(|e| match e { - Expr::Alias(a) => { - let expr = if temporal_input && is_temporal_output_column(&a.expr) { - Unresolved::Column(ColumnRef::Named("value".into())) - } else { - df_expr_to_unresolved(&a.expr)? - }; - Ok::, LoweringError>(ProjectItem { - expr, - alias: Some(a.name.clone()), - }) - } - _ => { - let expr = if temporal_input && is_temporal_output_column(e) { - Unresolved::Column(ColumnRef::Named("value".into())) - } else { - df_expr_to_unresolved(e)? - }; - Ok::, LoweringError>(ProjectItem { expr, alias: None }) - } - }) - .collect::, _>>()?; - Ok(Unresolved::Project { - cols, - qualifier: None, - child, - }) - } - - fn lower_aggregate(&self, agg: &logical_expr::Aggregate) -> Result { - let input = self.lower_plan(&agg.input)?; - // Each measure's row predicate (`FILTER (WHERE …)`, or the NULL-skip - // a `count(expr)` implies), read off the original typed expression - // before derived-column rewriting erases the argument's nullability. - let measure_filters = agg - .aggr_expr - .iter() - .map(|e| measure_filter(e, agg.input.schema())) - .collect::, LoweringError>>()?; - - if agg.aggr_expr.iter().any(is_temporal_aggregate) { - if measure_filters.iter().any(Option::is_some) { - return Err(LoweringError::UnsupportedFeature( - "FILTER on an ASAP temporal aggregate".into(), - )); - } - return self.lower_temporal_aggregate(agg, input); - } - - // `GROUPING SETS`/`ROLLUP`/`CUBE` emit several grouping levels from one - // scan. `Aggregate.by` is a single key set, so each level becomes its own - // `Aggregate` and they are merged (issue #118). - if let Some(gs) = agg.group_expr.iter().find_map(as_grouping_set) { - if measure_filters.iter().any(Option::is_some) { - return Err(LoweringError::UnsupportedFeature( - "FILTER on a measure inside a multi-level grouping".into(), - )); - } - return self.lower_grouping_sets(agg, gs, input); - } - - // `Aggregate.by` and the reducers index *columns*, so a grouping or - // reducer expression (`GROUP BY date_trunc(…)`, `SUM(a * 8)`) has no - // slot. Materialize each one as a derived column in a `Project` beneath - // the aggregate, then group/reduce over that column (issue #110). - let mut derived = DerivedCols::default(); - - // DataFusion strips `AS m` from a grouping expression, so the aggregate - // schema's field name is what the enclosing Projection references — - // the derived column has to carry exactly that name. - let group_names: Vec = agg - .schema - .fields() - .iter() - .take(agg.group_expr.len()) - .map(|f| f.name().to_string()) - .collect(); - - let mut keys = Vec::with_capacity(agg.group_expr.len()); - for (i, e) in agg.group_expr.iter().enumerate() { - match unalias(e) { - Expr::Column(_) => { - derived.passthrough(e)?; - keys.push(expr_to_group_ref(e)?); - } - other => { - let name = group_names - .get(i) - .cloned() - .unwrap_or_else(|| other.to_string()); - derived.materialize(name.clone(), df_expr_to_unresolved(other)?)?; - keys.push(ColumnRef::Named(name)); - } - } - } - - // Reducer arguments get the same treatment; `rewrite_agg` returns the - // aggregate with its argument repointed at the derived column. - let aggr_expr = agg - .aggr_expr - .iter() - .map(|e| derived.rewrite_agg(e)) - .collect::, LoweringError>>()?; - // A measure filter reads the aggregate's input rows, so the columns - // it names must survive any derived-column `Project` inserted below. - for column in measure_filters.iter().flatten().flat_map(Expr::column_refs) { - derived.passthrough(&Expr::Column(column.clone()))?; - } - - let child = Rc::new(derived.wrap(input)?); - // DataFusion names the aggregate outputs in its own schema (e.g. - // "sum(metrics.bytes)") — the same names the enclosing Projection - // references. The schema is [group fields …, aggregate fields …], so - // skip the group fields and thread the rest straight through as - // `Aggregate.output_names`, letting that Projection resolve them. - let output_names: Vec = agg - .schema - .fields() - .iter() - .skip(agg.group_expr.len()) - .map(|f| f.name().to_string()) - .collect(); - let measures = aggr_expr - .iter() - .map(lower_agg_intent) - .collect::, LoweringError>>()?; - // Empty when nothing is filtered — the one canonical unfiltered shape. - let filters = if measure_filters.iter().any(Option::is_some) { - measure_filters - .iter() - .map(|f| { - f.as_ref() - .map(|f| Ok(Predicate(Rc::new(df_expr_to_unresolved(f)?)))) - .transpose() - }) - .collect::, LoweringError>>()? - } else { - Vec::new() - }; - Ok(Unresolved::Aggregate { - // SQL `GROUP BY` is always an inclusion list, never PromQL's - // `without(...)` exclusion form — and always a genuine reduction, - // never `PerEntity` (there's no windowed/subquery-child concept - // in SQL for that to apply to). - reduction: Reduction::Reduce(GroupKeys::by(keys)), - measures, - output_names, - filters, - having: None, - child, - }) - } - - fn lower_temporal_aggregate( - &self, - agg: &logical_expr::Aggregate, - input: Unresolved, - ) -> Result { - if agg.aggr_expr.len() != 1 { - return Err(LoweringError::UnsupportedFeature( - "an ASAP temporal aggregate cannot share an Aggregate node with another reducer" - .into(), - )); - } - let Expr::AggregateFunction(call) = unalias(&agg.aggr_expr[0]) else { - unreachable!("is_temporal_aggregate accepted a non-aggregate expression") - }; - let name = call.func.name().to_lowercase(); - let [value, timestamp, window] = call.params.args.as_slice() else { - unreachable!("ASAP temporal UDAF signatures require exactly three arguments") - }; - - let value_ref = reducer_col(&name, std::slice::from_ref(value))?; - let timestamp_ref = reducer_col(&name, std::slice::from_ref(timestamp))?; - let Expr::Literal(window, _) = unalias(window) else { - return Err(LoweringError::InvalidExpression(format!( - "{name} window_ms must be a positive integer literal" - ))); - }; - let window_ms = scalar_positive_u64(window).ok_or_else(|| { - LoweringError::InvalidExpression(format!( - "{name} window_ms must be a positive integer literal" - )) - })?; - - let resolved_input = resolve_root(&input)?; - let input_schema = resolved_input.output_schema().map_err(|error| { - LoweringError::InvalidExpression(format!( - "cannot derive temporal aggregate input schema: {error}" - )) - })?; - let timestamp_id = resolve_column_ref(×tamp_ref, &input_schema).map_err(|error| { - LoweringError::InvalidExpression(format!("{name} timestamp argument: {error}")) - })?; - if input_schema.time_index != Some(timestamp_id) { - return Err(LoweringError::InvalidExpression(format!( - "{name} timestamp argument must name the input schema's time-index column" - ))); - } - let value_id = resolve_column_ref(&value_ref, &input_schema).map_err(|error| { - LoweringError::InvalidExpression(format!("{name} value argument: {error}")) - })?; - if value_id == timestamp_id - || !matches!( - input_schema.fields[value_id].dtype, - FieldDataType::Plain(DataType::Int64 | DataType::Float64) - ) - { - return Err(LoweringError::InvalidExpression(format!( - "{name} value argument must name a numeric non-time column" - ))); - } - - let mut group_ids = Vec::with_capacity(agg.group_expr.len()); - let mut group_refs = Vec::with_capacity(agg.group_expr.len()); - for group in &agg.group_expr { - let group_ref = expr_to_group_ref(group)?; - let group_id = resolve_column_ref(&group_ref, &input_schema).map_err(|error| { - LoweringError::InvalidExpression(format!("{name} GROUP BY column: {error}")) - })?; - if group_id == timestamp_id || group_id == value_id { - return Err(LoweringError::InvalidExpression(format!( - "{name} GROUP BY cannot contain its timestamp or value column" - ))); - } - if group_ids.contains(&group_id) { - return Err(LoweringError::InvalidExpression(format!( - "{name} GROUP BY contains the same resolved column more than once" - ))); - } - group_ids.push(group_id); - group_refs.push(group_ref); - } - // Minimal series-identity contract without adding SQL-only metadata to - // the shared Schema: a declared row-unique key must contain the time - // index, and removing that index yields the complete series key. The - // GROUP BY must match that key exactly. A unique key that omits time is - // only row identity and proves nothing about time-series continuity. - let identifies_one_series = input_schema - .unique_keys - .iter() - .filter(|key| key.contains(×tamp_id)) - .any(|key| { - let mut series_key: Vec<_> = key - .iter() - .copied() - .filter(|id| *id != timestamp_id) - .collect(); - series_key.sort_unstable(); - series_key.dedup(); - let mut grouped = group_ids.clone(); - grouped.sort_unstable(); - series_key == grouped - }); - if !identifies_one_series { - return Err(LoweringError::InvalidExpression(format!( - "{name} GROUP BY must exactly match a declared series identity (a unique key without the time index)" - ))); - } - - let mut cols = vec![ - ProjectItem { - alias: Some("ts".into()), - expr: Unresolved::Column(timestamp_ref.clone()), - }, - ProjectItem { - alias: Some("value".into()), - expr: Unresolved::Column(value_ref.clone()), - }, - ]; - for group_ref in group_refs { - let group_name = named_ref(&group_ref).to_string(); - cols.push(ProjectItem { - alias: Some(group_name), - expr: Unresolved::Column(group_ref), - }); - } - let child = Unresolved::Project { - cols, - qualifier: None, - child: Rc::new(input), - }; - let child = Unresolved::TimeRange { - range: Duration::from_millis(window_ms), - child: Rc::new(child), - }; - let intent = match name.as_str() { - "asap_rate" => AggIntent::Rate, - "asap_increase" => AggIntent::Increase, - - _ => unreachable!("is_temporal_aggregate admitted {name}"), - }; - Ok(Unresolved::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(child), - }) - } - - /// `GROUP BY ROLLUP/CUBE/GROUPING SETS` — multi-level grouping (issue #118). - /// - /// One scan produces several grouping levels; `Aggregate.by` holds a single - /// key set. So each level becomes its own `Aggregate`, and the levels are - /// `Concat`ed. A level that omits a key still has to *emit* it — as `NULL`, per - /// SQL — so each branch is wrapped in a `Project` that reinstates the missing - /// keys as typed nulls and restores the canonical column order. That keeps - /// the branches union-compatible, which `Concat` requires (it derives its - /// schema from the first child). - /// - /// `Aggregate.child` is duplicated per level — the same trade - /// `histogram_quantiles` makes (#109); a future workload-level reuse pass - /// could hoist it back into a single producer. - /// - /// DataFusion's `__grouping_id` discriminator is dropped: it only exists to - /// tell a subtotal's `NULL` apart from a data `NULL`, which is observable - /// solely through `GROUPING(col)` — an aggregate this front end rejects. - fn lower_grouping_sets( - &self, - agg: &logical_expr::Aggregate, - gs: &logical_expr::GroupingSet, - input: Unresolved, - ) -> Result { - // DataFusion normalizes every mixed form (`GROUP BY g, ROLLUP(d)`) into a - // single `GroupingSets`, so one grouping expression is the only shape. - if agg.group_expr.len() != 1 { - return Err(LoweringError::UnsupportedFeature( - "a grouping set alongside plain GROUP BY keys".into(), - )); - } - - // `distinct_expr()` is ordered exactly like the aggregate's leading - // schema fields, which is the column order the enclosing Projection - // expects. The field after them is `__grouping_id`. - let distinct = gs.distinct_expr(); - for e in &distinct { - if !matches!(unalias(e), Expr::Column(_)) { - return Err(LoweringError::UnsupportedFeature(format!( - "non-column key inside a multi-level grouping: {e}" - ))); - } - } - let keys: Vec<(String, DataType)> = agg - .schema - .fields() - .iter() - .take(distinct.len()) - .map(|f| Ok((f.name().to_string(), arrow_to_dtype(f.data_type())?))) - .collect::>()?; - - let output_names: Vec = agg - .schema - .fields() - .iter() - .skip(distinct.len() + 1) // + `__grouping_id` - .map(|f| f.name().to_string()) - .collect(); - - // Reducer arguments still materialize as derived columns (#110); the - // grouping keys are plain columns, so they only need carrying through. - let mut derived = DerivedCols::default(); - for e in &distinct { - derived.passthrough(e)?; - } - let aggr_expr = agg - .aggr_expr - .iter() - .map(|e| derived.rewrite_agg(e)) - .collect::, LoweringError>>()?; - let measures = aggr_expr - .iter() - .map(lower_agg_intent) - .collect::, LoweringError>>()?; - let input = derived.wrap(input)?; - - let branches = expand_grouping_set(gs) - .iter() - .map(|level| { - let level_keys = distinct - .iter() - .filter(|e| level.contains(e)) - .map(|e| expr_to_group_ref(e)) - .collect::, LoweringError>>()?; - let aggregate = Unresolved::Aggregate { - reduction: Reduction::Reduce(GroupKeys::by(level_keys)), - measures: measures.clone(), - output_names: output_names.clone(), - filters: vec![], - having: None, - child: Rc::new(input.clone()), - }; - // Reinstate omitted keys as typed nulls, in canonical order. - let cols = keys - .iter() - .zip(&distinct) - .map(|((name, dtype), e)| ProjectItem { - alias: Some(name.clone()), - expr: if level.contains(e) { - Unresolved::Column(ColumnRef::Named(name.clone())) - } else { - Unresolved::Cast { - expr: Rc::new(Unresolved::Literal(ScalarValue::Null)), - to: dtype.clone(), - try_cast: false, - } - }, - }) - .chain(output_names.iter().map(|n| ProjectItem { - alias: Some(n.clone()), - expr: Unresolved::Column(ColumnRef::Named(n.clone())), - })) - .collect(); - Ok(Unresolved::Project { - cols, - qualifier: None, - child: Rc::new(aggregate), - }) - }) - .collect::, LoweringError>>()?; - - // No discriminator asserted here today (issue #228): DataFusion's own - // `__grouping_id` would be the natural one, but this front end - // already discards it (see above — `GROUPING()` itself is rejected), - // so there is no distinct-per-branch column available to name yet. - // `Unresolved::concat` keeps `output_schema`'s default (drop - // `unique_keys` entirely). See - // `docs/design_docs/concat-unique-keys-decision.md`. - Ok(Unresolved::concat(branches)) - } - - fn lower_sort(&self, sort: &logical_expr::Sort) -> Result { - // A count-ranked `ORDER BY … LIMIT k` is the frequency heavy-hitter the - // `TopK` intent represents, but that promotion now happens in the shared - // `canonicalize` pass (issue #34) — the same one both front ends run — - // so SQL emits a plain `Sort` (+ `Limit`) here and lets canonicalization - // recognise the count-ranked shape positionally. This removes the gate's - // alias blind spot (#20). - let keys = sort - .expr - .iter() - .map(|s| { - df_expr_to_unresolved(&s.expr).map(|expr| SortKey { - expr, - ascending: s.asc, - nulls_first: s.nulls_first, - }) - }) - .collect::, _>>()?; - Ok(Unresolved::Sort { - keys, - // SQL `ORDER BY` is a global sort; per-group ranking would come from a - // window function (`SQLWindowFunc`), not a bare Sort. - partition_by: GroupKeys::none(), - child: Rc::new(self.lower_plan(&sort.input)?), - }) - } - - fn lower_limit(&self, limit: &logical_expr::Limit) -> Result { - // Count-ranked `LIMIT k` over a `Sort` is promoted to the heavy-hitter - // `TopK` by the shared `canonicalize` pass (issue #34), not here. - Ok(Unresolved::Limit { - n: eval_fetch(&limit.fetch).unwrap_or(usize::MAX), - offset: eval_fetch(&limit.skip).unwrap_or(0), - child: Rc::new(self.lower_plan(&limit.input)?), - }) - } -} - -/// A deliberately explicit marker accepted only in a projection of planning -/// SQL. The marker describes a relation operator, so it is removed rather than -/// lowered to the ordinary scalar `FunctionCall` variant. -enum PlanningBridge { - PromqlSubquery { - range: Duration, - resolution: Duration, - }, - HistogramQuantile { - q: f64, - }, -} - -fn planning_bridge( - projection: &logical_expr::Projection, -) -> Result, LoweringError> { - let mut found = None; - for expr in &projection.expr { - let Expr::ScalarFunction(call) = unalias(expr) else { - continue; - }; - let name = call.func.name().to_ascii_lowercase(); - let bridge = match name.as_str() { - "asap_promql_subquery" => { - let [range, resolution] = call.args.as_slice() else { - return Err(LoweringError::InvalidExpression( - "asap_promql_subquery requires (range_ms, resolution_ms)".into(), - )); - }; - let range = positive_millis_literal(range, "range_ms")?; - let resolution = positive_millis_literal(resolution, "resolution_ms")?; - PlanningBridge::PromqlSubquery { range, resolution } - } - "asap_histogram_quantile" => { - let [q] = call.args.as_slice() else { - return Err(LoweringError::InvalidExpression( - "asap_histogram_quantile requires one literal quantile".into(), - )); - }; - let q = float_literal(q).ok_or_else(|| { - LoweringError::InvalidExpression( - "asap_histogram_quantile quantile must be a numeric literal".into(), - ) - })?; - if !q.is_finite() || !(0.0..=1.0).contains(&q) { - return Err(LoweringError::InvalidExpression(format!( - "asap_histogram_quantile quantile must be finite and in [0,1], got {q}" - ))); - } - PlanningBridge::HistogramQuantile { q } - } - _ => continue, - }; - if found.is_some() { - return Err(LoweringError::InvalidExpression( - "a planning projection may contain only one asap_* relation marker".into(), - )); - } - found = Some(bridge); - } - if matches!(found, Some(PlanningBridge::HistogramQuantile { .. })) && projection.expr.len() != 1 - { - return Err(LoweringError::InvalidExpression( - "asap_histogram_quantile must be the projection's only expression".into(), - )); - } - Ok(found) -} - -/// Rebuild the SQL projection around the relation sampled by the temporal -/// marker. The marker's alias names the existing child column that occupies -/// its output slot (`... asap_promql_subquery(...) AS value ...`). This makes -/// the bridge schema-preserving without silently retaining columns that SQL -/// projected away. -fn temporal_bridge_projection( - projection: &logical_expr::Projection, - child: Unresolved, -) -> Result { - let cols = projection - .expr - .iter() - .map(|expr| { - if let Expr::ScalarFunction(call) = unalias(expr) { - if call - .func - .name() - .eq_ignore_ascii_case("asap_promql_subquery") - { - let Expr::Alias(alias) = expr else { - return Err(LoweringError::InvalidExpression( - "asap_promql_subquery must have an alias naming its child value column" - .into(), - )); - }; - return Ok(ProjectItem { - expr: Unresolved::Column(ColumnRef::Named(alias.name.clone())), - alias: Some(alias.name.clone()), - }); - } - } - match expr { - Expr::Alias(alias) => Ok(ProjectItem { - expr: df_expr_to_unresolved(&alias.expr)?, - alias: Some(alias.name.clone()), - }), - other => Ok(ProjectItem { - expr: df_expr_to_unresolved(other)?, - alias: None, - }), - } - }) - .collect::, LoweringError>>()?; - Ok(Unresolved::Project { - cols, - qualifier: None, - child: Rc::new(child), - }) -} - -fn positive_millis_literal(expr: &Expr, argument: &str) -> Result { - let millis = match unalias(expr) { - Expr::Literal(DfScalarValue::Int64(Some(value)), _) if *value > 0 => *value as u64, - Expr::Literal(DfScalarValue::UInt64(Some(value)), _) if *value > 0 => *value, - Expr::Literal(DfScalarValue::Int32(Some(value)), _) if *value > 0 => *value as u64, - other => { - return Err(LoweringError::InvalidExpression(format!( - "{argument} must be a positive integer millisecond literal, got {other}" - ))) - } - }; - Ok(Duration::from_millis(millis)) -} - -fn float_literal(expr: &Expr) -> Option { - match unalias(expr) { - Expr::Literal(DfScalarValue::Float64(Some(value)), _) => Some(*value), - Expr::Literal(DfScalarValue::Float32(Some(value)), _) => Some(*value as f64), - Expr::Literal(DfScalarValue::Int64(Some(value)), _) => Some(*value as f64), - Expr::Literal(DfScalarValue::UInt64(Some(value)), _) => Some(*value as f64), - Expr::Literal(DfScalarValue::Int32(Some(value)), _) => Some(*value as f64), - _ => None, - } -} - -// ── ClickHouse-builtin compatibility, taught to DataFusion itself ────────────── -// -// Generalized over `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` (issue -// #225): adding support for one more ClickHouse-only builtin DataFusion -// doesn't know at all is a catalog data entry (name, arity, `RewriteKind`) -// plus, only if its rewrite target is a genuinely new shape, one match arm -// in `ClickHouseBuiltinRewrite::rewrite` below — never a new stub-UDAF -// constructor or a new `FunctionRewrite`-implementing type. `uniqExact` -// (issue #221) and `countIf` both go through this one mechanism. - -/// A stub `AggregateUDF` for one `CLICKHOUSE_BUILTINS` entry, registered -/// purely so DataFusion's planner can resolve the function name during -/// `SqlToRel` conversion (it errors on an unknown function before a rewrite -/// ever gets a chance to run). Every call site is replaced by -/// `ClickHouseBuiltinRewrite` — via the `Analyzer` `lower()` runs after -/// parsing — before physical planning could ever ask this UDAF for an -/// `Accumulator`, so `accumulator` is unreachable for every catalog entry. -fn clickhouse_builtin_stub_udaf(name: &'static str, arity: Arity) -> AggregateUDF { - AggregateUDF::from(SimpleAggregateUDF::new_with_signature( - name, - arity_to_signature(arity), - ArrowDataType::Int64, - Arc::new(move |_| { - // ponytail: dead code by construction (see doc comment above) — - // a real accumulator would just reimplement whatever native - // shape `ClickHouseBuiltinRewrite` rewrites this call to. - unimplemented!( - "{name} has no accumulator: every call site is rewritten to a native \ - DataFusion aggregate before physical planning" - ) - }), - vec![], - )) -} - -/// A catalog [`Arity`] as the DataFusion `Signature` a stub UDAF/UDF is -/// registered with — shared by the aggregate stub above and the scalar stub -/// below, since neither wants to model per-argument types, only how many -/// arguments a call may take. -fn arity_to_signature(arity: Arity) -> Signature { - match arity { - Arity::Exact(n) => Signature::any(n, Volatility::Immutable), - Arity::Range { min, max } => Signature::one_of( - (min..=max).map(TypeSignature::Any).collect(), - Volatility::Immutable, - ), - } -} - -// ── ClickHouse scalar-builtin compatibility ───────────────────────────────── -// -// The scalar counterpart of the aggregate mechanism above, but simpler: -// `asap_sql_function_catalog::CLICKHOUSE_SCALAR_BUILTINS` carries no -// `RewriteKind`, because a scalar call needs none. Unlike an aggregate call -// (which must become a real `AggIntent`, hence the rewrite to a native -// DataFusion aggregate shape `lower_agg_intent` can classify), a scalar -// function call in this IR is already deliberately opaque — -// `expr::df_expr_to_unresolved`'s `Expr::ScalarFunction` arm lowers *any* -// scalar call generically to `Unresolved::FunctionCall { name, args }`, with -// zero name-specific logic. So teaching DataFusion's planner to accept a -// ClickHouse scalar builtin's name — a stub `ScalarUDF`, registered below — -// is the entire fix; the existing generic lowering already does the rest. - -/// A stub `ScalarUDF` for one `CLICKHOUSE_SCALAR_BUILTINS` entry, registered -/// purely so DataFusion's planner can resolve the function name during -/// `SqlToRel` conversion (it errors on an unknown function otherwise), and so -/// it can keep building the surrounding expression's type from a plausible -/// return type. Unlike `clickhouse_builtin_stub_udaf`, no `FunctionRewrite` -/// ever fires for these — the call survives to `lower_plan` as-is and lowers -/// through the generic `Expr::ScalarFunction` arm — so `invoke`/`invoke_batch` -/// (left at their default, which returns a `NotImplemented` `DataFusionError`) -/// are unreachable for every catalog entry: this front end only ever uses -/// DataFusion for planning/type-checking, never physical execution. -fn clickhouse_scalar_builtin_stub_udf(name: &'static str, arity: Arity) -> ScalarUDF { - ScalarUDF::from(ClickHouseScalarBuiltinStub { - name, - signature: arity_to_signature(arity), - return_type: clickhouse_scalar_builtin_return_type(name), - }) -} - -/// A plausible Arrow return type for one `CLICKHOUSE_SCALAR_BUILTINS` entry — -/// just precise enough that DataFusion's planner can keep building the type -/// of whatever expression the call sits inside (e.g. a `WHERE` predicate -/// wants `Boolean`), not a claim about ClickHouse's actual return type. -/// Real function typing happens downstream, at post-ASAP binding. -fn clickhouse_scalar_builtin_return_type(name: &str) -> ArrowDataType { - match name { - // Array(String) in ClickHouse; a plain `Utf8` element list is close - // enough for planning purposes here. - "splitbychar" => ArrowDataType::List(Arc::new(datafusion::arrow::datatypes::Field::new( - "item", - ArrowDataType::Utf8, - true, - ))), - "todate" => ArrowDataType::Date32, - // ClickHouse returns UInt8 (0/1), but every corpus use is a boolean - // predicate — `Boolean` keeps that context type-checking. - "match" | "startswith" => ArrowDataType::Boolean, - "tostartofhour" - | "tostartofweek" - | "tostartofminute" - | "tostartoffiveminutes" - | "tostartofinterval" => { - ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) - } - // 1-based match position, 0 if not found. - "positioncaseinsensitive" => ArrowDataType::UInt64, - // Relation markers are removed by `lower_projection`; Float64 merely - // lets DataFusion type the temporary SELECT list. - "asap_promql_subquery" | "asap_histogram_quantile" => ArrowDataType::Float64, - other => unreachable!( - "{other}: every CLICKHOUSE_SCALAR_BUILTINS entry must have a return type listed here" - ), - } -} - -/// A stub `ScalarUDFImpl` carrying only what DataFusion's planner needs: -/// name, arity-only [`Signature`], and a fixed return type. `invoke_with_args` -/// returns a `NotImplemented` `DataFusionError` — see -/// [`clickhouse_scalar_builtin_stub_udf`]'s doc for why that is unreachable in -/// practice. -#[derive(Debug, PartialEq, Eq, Hash)] -struct ClickHouseScalarBuiltinStub { - name: &'static str, - signature: Signature, - return_type: ArrowDataType, -} - -impl ScalarUDFImpl for ClickHouseScalarBuiltinStub { - fn name(&self) -> &str { - self.name - } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn return_type( - &self, - _arg_types: &[ArrowDataType], - ) -> datafusion::common::Result { - Ok(self.return_type.clone()) - } - - fn invoke_with_args( - &self, - _args: ScalarFunctionArgs, - ) -> datafusion::common::Result { - Err(datafusion::common::DataFusionError::NotImplemented( - format!("{} is a planning-only stub", self.name), - )) - } -} - -// ── ClickHouse window-builtin compatibility ───────────────────────────────── -// -// The window counterpart of the scalar mechanism above: a stub `WindowUDF` -// registered purely so DataFusion's planner accepts the call name during -// `SqlToRel` conversion. No rewrite step follows — `lower_window_func_kind` -// already maps each `asap_sql_function_catalog::CLICKHOUSE_WINDOW_BUILTINS` -// name directly to its own `WindowFuncKind` variant (issue #267). - -/// A stub `WindowUDF` for one `CLICKHOUSE_WINDOW_BUILTINS` entry, registered -/// purely so DataFusion's planner can resolve the function name inside an -/// `OVER (...)` clause. This front end only ever uses DataFusion for -/// planning/type-checking, never physical execution, so -/// `partition_evaluator` (which physical execution alone would call) is -/// unreachable in practice. -fn clickhouse_window_builtin_stub_udwf(name: &'static str, arity: Arity) -> WindowUDF { - WindowUDF::from(ClickHouseWindowBuiltinStub { - name, - signature: arity_to_signature(arity), - }) -} - -/// A stub `WindowUDFImpl` carrying only what DataFusion's planner needs: -/// name, arity-only [`Signature`], and a field type derived from the first -/// argument (matching `lag`/`lead`'s own "output type = input type" -/// behavior). `partition_evaluator` is left `unimplemented!()` — see -/// [`clickhouse_window_builtin_stub_udwf`]'s doc for why that is unreachable. -#[derive(Debug, PartialEq, Eq, Hash)] -struct ClickHouseWindowBuiltinStub { - name: &'static str, - signature: Signature, -} - -impl WindowUDFImpl for ClickHouseWindowBuiltinStub { - fn name(&self) -> &str { - self.name - } - - fn signature(&self) -> &Signature { - &self.signature - } - - fn field(&self, field_args: WindowUDFFieldArgs) -> datafusion::common::Result { - let dtype = field_args - .get_input_field(0) - .map_or(ArrowDataType::Null, |field| field.data_type().clone()); - Ok(Arc::new(Field::new(field_args.name(), dtype, true))) - } - - fn partition_evaluator( - &self, - _partition_evaluator_args: PartitionEvaluatorArgs, - ) -> datafusion::common::Result> { - let name = self.name; - unimplemented!( - "{name} has no partition evaluator: this front end never runs DataFusion's \ - physical planner, only SqlToRel + the unoptimized LogicalPlan" - ) - } -} - -/// Rewrites every `asap_sql_function_catalog::CLICKHOUSE_BUILTINS` call to -/// the native DataFusion aggregate shape its entry's `RewriteKind` names — -/// so a ClickHouse-only builtin DataFusion doesn't know at all becomes an -/// ordinary DataFusion aggregate before the plan ever reaches -/// `lower_agg_intent`, which needs no ClickHouse-specific name of its own. -#[derive(Debug)] -struct ClickHouseBuiltinRewrite; - -impl FunctionRewrite for ClickHouseBuiltinRewrite { - fn name(&self) -> &str { - "clickhouse builtin -> native DataFusion aggregate" - } - - fn rewrite( - &self, - expr: Expr, - _schema: &DFSchema, - _config: &ConfigOptions, - ) -> datafusion::common::Result> { - let Expr::AggregateFunction(f) = expr else { - return Ok(Transformed::no(expr)); - }; - let Some(builtin) = asap_sql_function_catalog::lookup_clickhouse_builtin(f.func.name()) - else { - return Ok(Transformed::no(Expr::AggregateFunction(f))); - }; - let rewritten = match builtin.rewrite { - // No native DataFusion shape to become — leave the call exactly - // as DataFusion's planner parsed it. `lower_agg_intent` handles - // the ClickHouse name (`argMax`/`argMin`) directly (issue #232). - RewriteKind::PassThrough => return Ok(Transformed::no(Expr::AggregateFunction(f))), - // `f(args...)` -> `count(args...) DISTINCT` — `lower_agg_intent` - // already maps `count` + `DISTINCT` to `AggIntent::Cardinality`, - // at whatever arity the call carries. - RewriteKind::CountDistinct => AggregateFunction::new_udf( - count_udaf(), - f.params.args, - true, - f.params.filter, - f.params.order_by, - f.params.null_treatment, - ), - // `f(cond)` -> `sum(CASE WHEN cond THEN 1 ELSE 0 END)` — see - // `RewriteKind::CountIfToSum`'s doc; moving the `-If` family onto - // `Aggregate.filters` (issue #466) is a follow-up. - RewriteKind::CountIfToSum => { - let cond = f.params.args.into_iter().next().expect( - "countif's stub signature fixes its arity at 1 -- the planner \ - already rejected any other argument count before this rewrite runs", - ); - let indicator = Expr::Case(Case::new( - None, - vec![(Box::new(cond), Box::new(lit(1i64)))], - Some(Box::new(lit(0i64))), - )); - AggregateFunction::new_udf( - sum_udaf(), - vec![indicator], - false, - f.params.filter, - f.params.order_by, - f.params.null_treatment, - ) - } - }; - Ok(Transformed::yes(Expr::AggregateFunction(rewritten))) - } -} - -// ── Aggregate / group-key helpers ─────────────────────────────────────────────── - -/// The row predicate one aggregate call carries (issue #466): its explicit -/// `FILTER (WHERE p)`, plus — for a plain `count(expr)`, which canonical -/// `AggIntent::Count` lowers to a row count that never looks at `expr` — the -/// NULL-skipping SQL gives it. `count(CASE WHEN p THEN x END)` is the -/// conditional-count idiom, so it becomes `p [AND x IS NOT NULL]` rather -/// than the opaque `CASE … IS NOT NULL`; any other nullable argument becomes -/// `expr IS NOT NULL`. `None` when the call updates on every row. -fn measure_filter(expr: &Expr, input: &DFSchema) -> Result, LoweringError> { - let Expr::AggregateFunction(agg_fn) = unalias(expr) else { - return Ok(None); - }; - let mut conjuncts: Vec = agg_fn.params.filter.iter().map(|f| (**f).clone()).collect(); - let counts_rows = agg_fn.func.name().eq_ignore_ascii_case("count") && !agg_fn.params.distinct; - if counts_rows { - for argument in &agg_fn.params.args { - let nullable = argument - .nullable(input) - .map_err(|error| LoweringError::UnsupportedFeature(error.to_string()))?; - if !nullable { - continue; - } - match conditional_count_arm(argument) { - Some((when, then)) => { - conjuncts.push(when.clone()); - if then - .nullable(input) - .map_err(|error| LoweringError::UnsupportedFeature(error.to_string()))? - { - conjuncts.push(then.clone().is_not_null()); - } - } - None => conjuncts.push(argument.clone().is_not_null()), - } - } - } - Ok(conjuncts.into_iter().reduce(Expr::and)) -} - -/// `CASE WHEN p THEN x END` (searched, one arm, no `ELSE` or `ELSE NULL`) -/// as `(p, x)`. -fn conditional_count_arm(expr: &Expr) -> Option<(&Expr, &Expr)> { - let Expr::Case(case) = unalias(expr) else { - return None; - }; - if case.expr.is_some() { - return None; - } - let else_is_null = match case.else_expr.as_deref() { - None => true, - Some(Expr::Literal(value, _)) => value.is_null(), - Some(_) => false, - }; - if !else_is_null { - return None; - } - let [(when, then)] = case.when_then_expr.as_slice() else { - return None; - }; - Some((when, then)) -} - -/// Map a DataFusion aggregate expression directly to the canonical -/// [`AggIntent`] — issue #179's "dedicated function → canonical -/// intent directly" front-end construction, no `AggFunc` intermediate. The -/// name → semantic mapping itself lives in `asap_sql_function_catalog` -/// (issue #225) as flat data (`NATIVE_FUNCTIONS`); what stays here is -/// call-site logic that isn't a function of the name alone — the DISTINCT -/// modifier rule, the "reducer argument must be a bare column" rule -/// (`reducer_col`), φ extraction from a literal argument, and the ambient -/// `AccuracyTarget`. `resolve_root` resolves `col` to a positional -/// `ColumnId`; the output name (DataFusion's own, e.g. -/// `"sum(metrics.bytes)"`) is threaded separately as `Aggregate.output_names`, -/// not carried here. -fn lower_agg_intent(expr: &Expr) -> Result, LoweringError> { - match expr { - Expr::Alias(a) => lower_agg_intent(&a.expr), - Expr::AggregateFunction(agg_fn) => { - let name = agg_fn.func.name().to_lowercase(); - // ClickHouse's row-selecting `argMax`/`argMin` — `RewriteKind:: - // PassThrough` in the catalog, so the call reaches here under its - // own name rather than a native DataFusion aggregate. Handled - // before the `NATIVE_FUNCTIONS` lookup below since neither name - // is in that table (issue #232). - if let Some(intent) = lower_arg_selector(&name, &agg_fn.params.args)? { - return Ok(intent); - } - let semantic = asap_sql_function_catalog::lookup_native(&name) - .ok_or_else(|| LoweringError::UnsupportedAggregate(name.clone()))?; - // The canonical intent algebra has no DISTINCT modifier for the - // value reducers; only - // COUNT(DISTINCT) maps (to Cardinality). Reject DISTINCT elsewhere - // rather than silently lowering `SUM(DISTINCT x)` as `SUM(x)`. - if agg_fn.params.distinct && !matches!(semantic, AggSemantic::Count) { - return Err(LoweringError::UnsupportedAggregate(format!( - "DISTINCT {name}" - ))); - } - // Value reducers (`reducer_col`) require a real column — `SUM(a*b)` - // is rejected, not silently reduced over a probe column. Quantile - // and CountDistinct reduce a column too, so they take the same path: - // `col` is `Option` once resolved, where `None` means "the - // PromQL sample value", which a SQL query never has. Taking an - // expression here would set `col: None` and silently drop it (#115). - let col = |args: &[Expr]| -> Result, LoweringError> { - reducer_col(&name, args).map(Some) - }; - Ok(match semantic { - AggSemantic::Correlation => { - if !agg_fn.params.order_by.is_empty() || agg_fn.params.null_treatment.is_some() - { - return Err(LoweringError::UnsupportedAggregate( - "corr with ORDER BY or explicit null treatment".into(), - )); - } - let [left, right] = agg_fn.params.args.as_slice() else { - return Err(LoweringError::UnsupportedAggregate( - "corr requires two arguments".into(), - )); - }; - AggIntent::PearsonCorr { - left: expr_to_group_ref(left)?, - right: expr_to_group_ref(right)?, - } - } - // Every argument reaches the intent: `COUNT(DISTINCT a, b)` - // counts distinct *tuples*, which is a different quantity from - // the distinct count of either column. - AggSemantic::Count if agg_fn.params.distinct => match agg_fn.params.args.as_slice() - { - // DataFusion's planner rejects a bare `COUNT(DISTINCT)` - // before lowering. Guarded anyway: an empty `cols` is the - // PromQL sample-value convention, which SQL never has. - [] => { - return Err(LoweringError::UnsupportedAggregate( - "COUNT(DISTINCT) without an argument".into(), - )) - } - args => AggIntent::Cardinality { - cols: args.iter().map(distinct_col).collect::>()?, - accuracy: current_accuracy(), - }, - }, - AggSemantic::Count => AggIntent::Count { - accuracy: current_accuracy(), - }, - AggSemantic::Sum => AggIntent::Sum { - col: col(&agg_fn.params.args)?, - }, - AggSemantic::Min => AggIntent::Min { - col: col(&agg_fn.params.args)?, - }, - AggSemantic::Max => AggIntent::Max { - col: col(&agg_fn.params.args)?, - }, - AggSemantic::Avg => AggIntent::Avg { - col: col(&agg_fn.params.args)?, - }, - AggSemantic::StdDev { population } => AggIntent::StdDev { - col: col(&agg_fn.params.args)?, - population, - }, - AggSemantic::Variance { population } => AggIntent::Variance { - col: col(&agg_fn.params.args)?, - population, - }, - // `fixed_q = Some(0.5)` is `median`/`approx_median`. As with - // `approx_distinct` and `approx_percentile_cont`, the - // `approx_` prefix does not force an approximation: the - // sketch-vs-exact choice is the AccuracyTarget's (see - // `plan::boundary`), so both spellings share one intent - // (#111). - AggSemantic::Quantile { fixed_q } => AggIntent::Quantile { - col: col(&agg_fn.params.args)?, - q: match fixed_q { - Some(q) => q, - None => extract_percentile_q(&agg_fn.params.args)?, - }, - accuracy: current_accuracy(), - }, - AggSemantic::Cardinality => AggIntent::Cardinality { - cols: vec![reducer_col(&name, &agg_fn.params.args)?], - accuracy: current_accuracy(), - }, - }) - } - _ => Err(LoweringError::UnsupportedAggregate(format!( - "measure is not an aggregate function call: {expr}" - ))), - } -} - -fn temporal_aggregate_name(expr: &Expr) -> Option { - let Expr::AggregateFunction(call) = unalias(expr) else { - return None; - }; - let name = call.func.name().to_lowercase(); - matches!(name.as_str(), "asap_rate" | "asap_increase").then_some(name) -} - -fn is_temporal_aggregate(expr: &Expr) -> bool { - temporal_aggregate_name(expr).is_some() -} - -fn is_temporal_output_column(expr: &Expr) -> bool { - let Expr::Column(col) = unalias(expr) else { - return false; - }; - let name = col.name.to_lowercase(); - ["asap_rate(", "asap_increase("] - .iter() - .any(|prefix| name.starts_with(prefix)) -} - -fn plan_has_temporal_aggregate(plan: &LogicalPlan) -> bool { - match plan { - LogicalPlan::Aggregate(agg) => agg.aggr_expr.iter().any(is_temporal_aggregate), - LogicalPlan::Filter(filter) => plan_has_temporal_aggregate(&filter.input), - LogicalPlan::SubqueryAlias(alias) => plan_has_temporal_aggregate(&alias.input), - _ => false, - } -} - -fn named_ref(col: &ColumnRef) -> &str { - match col { - ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => name, - ColumnRef::SampleValue | ColumnRef::Wildcard => { - unreachable!("reducer_col only returns named column references") - } - } -} - -fn scalar_positive_u64(value: &DfScalarValue) -> Option { - match value { - DfScalarValue::Int64(Some(v)) if *v > 0 => Some(*v as u64), - DfScalarValue::Int32(Some(v)) if *v > 0 => Some(*v as u64), - DfScalarValue::UInt64(Some(v)) if *v > 0 => Some(*v), - DfScalarValue::UInt32(Some(v)) if *v > 0 => Some(*v as u64), - _ => None, - } -} - -/// ClickHouse's row-selecting `argMax(arg, val)` / `argMin(arg, val)` — -/// "return `arg`'s value from the row where `val` is maximal/minimal". -/// `Some(name)` for `"argmax"`/`"argmin"`, `None` for every other name (the -/// caller falls through to the ordinary `NATIVE_FUNCTIONS` path). -/// -/// Unlike every existing `AggIntent` reducer (`Sum`/`Min`/`Max`/`Avg`/…), -/// which folds *one* column to a value derived from itself, this is a -/// two-column, row-selecting aggregate: it returns a *different* column's -/// value, selected by which row maximizes/minimizes a second column. No -/// existing `AggIntent` shape fits, and — per its own doc comment's -/// "core only grows for intents ≥2 deployment models actually use" bar — -/// a repo-wide search (PromQL front end, the other SQL dialects, docs) found -/// no second deployment model wanting this shape, so this lowers to -/// `AggIntent::Extension` rather than earning a first-class `ArgMax`/`ArgMin` -/// core variant (issue #232). Core treats `Extension` opaquely: both columns -/// are kept only as validated bare-column names in `payload` (`reducer_col`'s -/// same "no expression arguments" rule, issue #115) — they are **not** run -/// through `resolve_agg_intent`'s positional `ColumnRef` -> `ColumnId` -/// binding the way a real reducer's `col` is, since `Extension` carries no -/// typed column field for core to resolve. Shared `arg_selector_columns` validates -/// and resolves those names during aggregate schema derivation, preserving the -/// selected argument's type and nullability for downstream exact execution. -/// -/// DerivedCols preserves both bare-column arguments when grouping expressions -/// introduce an intermediate Project. Shared aggregate schema derivation resolves -/// the payload and preserves the selected argument's type and nullability. -fn lower_arg_selector( - name: &str, - args: &[Expr], -) -> Result>, LoweringError> { - let ext_kind = match name { - "argmax" => "arg_max", - "argmin" => "arg_min", - _ => return Ok(None), - }; - let [arg, val] = args else { - unreachable!( - "{name}'s stub signature (asap_sql_function_catalog::CLICKHOUSE_BUILTINS) fixes \ - its arity at 2 -- the planner already rejected any other argument count before \ - lower_agg_intent runs" - ); - }; - let arg_col = reducer_col(name, std::slice::from_ref(arg))?; - let val_col = reducer_col(name, std::slice::from_ref(val))?; - Ok(Some(AggIntent::Extension { - ext_kind: ext_kind.to_string(), - payload: serde_json::json!({ "arg_col": arg_col, "val_col": val_col }), - })) -} - -/// The name an `IN (subquery)`'s key column is projected under, so the join -/// predicate cannot bind it to a same-named column of the outer relation. -const IN_SUBQUERY_KEY: &str = "__asap_in_key"; - -/// Fold `pred` directly onto `child.predicates` when `child` is a bare `Scan` -/// (a `WHERE` directly over a table), otherwise wrap it in an ordinary -/// `Filter` — canonical's invariant that a `Filter` never sits directly over a -/// `Scan`. A front end emitting the canonical shape directly is responsible -/// for maintaining that invariant itself (issue #179). -fn filter_or_fold(pred: Unresolved, child: Unresolved) -> Unresolved { - match child { - Unresolved::Scan { - source, - mut predicates, - schema, - } => { - predicates.push(Predicate(Rc::new(pred))); - Unresolved::Scan { - source, - predicates, - schema, - } - } - other => Unresolved::Filter { - pred: Predicate(Rc::new(pred)), - child: Rc::new(other), - }, - } -} - -/// Flatten a top-level `AND` chain into its conjuncts. -fn split_conjunction<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { - match expr { - Expr::BinaryExpr(b) if b.op == logical_expr::Operator::And => { - split_conjunction(&b.left, out); - split_conjunction(&b.right, out); - } - other => out.push(other), - } -} - -/// Re-`AND` the conjuncts, or `None` when there are none left. -fn rebuild_conjunction(conjuncts: &[&Expr]) -> Option { - conjuncts - .iter() - .map(|e| (*e).clone()) - .reduce(|acc, e| acc.and(e)) -} - -/// Split a correlated subquery's plan into `(uncorrelated plan, correlation)`. -/// -/// The correlation is the conjunction of the filter conjuncts that mention an -/// outer column, rewritten so `outer_ref(t.c)` becomes a plain `t.c` — it then -/// resolves against the join's concatenated `left ++ right` schema, like any -/// other join predicate. Everything else stays an ordinary inner `Filter`. -/// -/// An outer reference anywhere but a top-level filter conjunct is rejected: it -/// would need real decorrelation, not a predicate lift. -fn split_correlation(plan: &LogicalPlan) -> Result<(LogicalPlan, Option), LoweringError> { - let LogicalPlan::Filter(filter) = plan else { - return if plan_has_outer_ref(plan) { - Err(LoweringError::UnsupportedFeature( - "correlated subquery whose outer reference is not a filter conjunct".into(), - )) - } else { - Ok((plan.clone(), None)) - }; - }; - - let mut conjuncts = Vec::new(); - split_conjunction(&filter.predicate, &mut conjuncts); - let (correlated, inner): (Vec<_>, Vec<_>) = - conjuncts.into_iter().partition(|e| expr_has_outer_ref(e)); - - let input = filter.input.as_ref(); - if plan_has_outer_ref(input) { - return Err(LoweringError::UnsupportedFeature( - "correlated subquery whose outer reference is below its filter".into(), - )); - } - - let correlation = rebuild_conjunction(&correlated) - .map(|e| strip_outer_refs(&e)) - .transpose()?; - let plan = match rebuild_conjunction(&inner) { - Some(pred) => LogicalPlan::Filter( - logical_expr::Filter::try_new(pred, filter.input.clone()) - .map_err(LoweringError::DataFusion)?, - ), - None => input.clone(), - }; - Ok((plan, correlation)) -} - -/// Rewrite `outer_ref(t.c)` to `t.c` so the expression resolves against the -/// join's concatenated schema. -fn strip_outer_refs(expr: &Expr) -> Result { - expr.clone() - .transform(|e| { - Ok(match e { - Expr::OuterReferenceColumn(_, col) => Transformed::yes(Expr::Column(col)), - other => Transformed::no(other), - }) - }) - .map(|t| t.data) - .map_err(LoweringError::DataFusion) -} - -fn expr_has_outer_ref(expr: &Expr) -> bool { - let mut found = false; - expr.apply(|e| { - if matches!(e, Expr::OuterReferenceColumn(..)) { - found = true; - return Ok(TreeNodeRecursion::Stop); - } - Ok(TreeNodeRecursion::Continue) - }) - .expect("infallible visitor"); - found -} - -fn plan_has_outer_ref(plan: &LogicalPlan) -> bool { - let mut found = false; - plan.apply(|p| { - if p.expressions().iter().any(expr_has_outer_ref) { - found = true; - return Ok(TreeNodeRecursion::Stop); - } - Ok(TreeNodeRecursion::Continue) - }) - .expect("infallible visitor"); - found -} - -/// Strip `AS alias` wrappers. -fn unalias(expr: &Expr) -> &Expr { - match expr { - Expr::Alias(a) => unalias(&a.expr), - other => other, - } -} - -/// The `GroupingSet` inside a grouping expression, if any. -fn as_grouping_set(expr: &Expr) -> Option<&logical_expr::GroupingSet> { - match unalias(expr) { - Expr::GroupingSet(gs) => Some(gs), - _ => None, - } -} - -/// The grouping levels a `GroupingSet` stands for, widest first (issue #118). -/// -/// `ROLLUP(a, b)` → `(a,b), (a), ()` — the prefixes. -/// `CUBE(a, b)` → `(a,b), (a), (b), ()` — the power set. -/// `GROUPING SETS` is already the explicit list. -fn expand_grouping_set(gs: &logical_expr::GroupingSet) -> Vec> { - match gs { - logical_expr::GroupingSet::Rollup(exprs) => (0..=exprs.len()) - .rev() - .map(|n| exprs[..n].to_vec()) - .collect(), - logical_expr::GroupingSet::Cube(exprs) => { - // Bitmask descending, so the full set leads and `()` trails. - (0..(1u32 << exprs.len())) - .rev() - .map(|mask| { - exprs - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, e)| e.clone()) - .collect() - }) - .collect() - } - logical_expr::GroupingSet::GroupingSets(sets) => sets.clone(), - } -} - -/// Derived columns materialized in a `Project` beneath an `Aggregate` (#110). -/// -/// `Aggregate.by` holds positional `ColumnId`s and each reducer holds one input -/// column, so neither can hold an expression. `GROUP BY date_trunc('minute', t)` -/// and `SUM(bytes * 8)` are therefore rewritten to group/reduce over a projected -/// column that carries the expression's value. -/// -/// The projection also has to carry through the plain columns the aggregate -/// still references, since a `Project` replaces its child's schema rather than -/// extending it. -#[derive(Default)] -struct DerivedCols { - cols: Vec>, - /// Whether any column is genuinely derived. Without one the aggregate keeps - /// its original child, so DAGs that lower today keep their exact shape. - any: bool, - /// First same-name-different-value collision, reported only if the - /// projection is actually inserted (see [`Self::wrap`]). - collision: Option, -} - -impl DerivedCols { - /// Add `alias := expr`, or note a collision if `alias` already means - /// something else. `Project` carries one relation qualifier for all its - /// columns, so `a.k` and `b.k` cannot both survive it — but that only - /// matters when a projection gets inserted at all. - fn push(&mut self, alias: String, expr: Unresolved) { - let existing = self - .cols - .iter() - .find(|c| c.alias.as_deref() == Some(&alias)); - match existing { - // Same name, same value — one projected column serves both uses. - Some(e) if e.expr == expr => {} - Some(_) => { - self.collision.get_or_insert(alias); - } - None => self.cols.push(ProjectItem { - alias: Some(alias), - expr, - }), - } - } - - /// A plain column the aggregate references — carried through unchanged. - fn passthrough(&mut self, expr: &Expr) -> Result<(), LoweringError> { - let Expr::Column(c) = unalias(expr) else { - return Ok(()); - }; - self.push(c.name.clone(), df_expr_to_unresolved(expr)?); - Ok(()) - } - - /// A genuinely derived column: `alias` now names `expr`'s value. - fn materialize(&mut self, alias: String, expr: Unresolved) -> Result<(), LoweringError> { - self.any = true; - self.push(alias, expr); - Ok(()) - } - - /// Repoint a reducer's argument at a derived column when it is an - /// expression; otherwise carry its plain input column through. - fn rewrite_agg(&mut self, expr: &Expr) -> Result { - let Expr::AggregateFunction(agg_fn) = unalias(expr) else { - return Ok(expr.clone()); - }; - if matches!( - asap_sql_function_catalog::lookup_native(&agg_fn.func.name().to_lowercase()), - Some(AggSemantic::Correlation) - ) { - // Give each value argument its own projected name, including casts - // and qualified columns. This retains both inputs and avoids losing - // relation qualifiers when the projection becomes an unqualified schema. - let mut rewritten = agg_fn.clone(); - for arg in &mut rewritten.params.args { - let alias = unalias(arg).to_string(); - self.materialize(alias.clone(), df_expr_to_unresolved(arg)?)?; - *arg = Expr::Column(DfColumn::new_unqualified(alias)); - } - return Ok(Expr::AggregateFunction(rewritten)); - } - // `COUNT(*)` reduces no column; `agg_col_name` covers bare/aliased/cast - // columns, so `None` here means the argument really is an expression. - let counts_rows = - agg_fn.func.name().eq_ignore_ascii_case("count") && !agg_fn.params.distinct; - let Some(arg) = agg_fn.params.args.first() else { - return Ok(expr.clone()); - }; - if counts_rows { - return Ok(expr.clone()); - } - // Preserve every additional column dependency (e.g. argMax's ordering - // column) when an unrelated grouping expression creates a Project. - // Literal parameters need no source column and remain untouched. - for argument in agg_fn.params.args.iter().skip(1) { - self.passthrough(argument)?; - } - match agg_col_name(&agg_fn.params.args) { - Some(name) => { - self.push(name, df_expr_to_unresolved(arg)?); - Ok(expr.clone()) - } - None => { - let alias = unalias(arg).to_string(); - self.materialize(alias.clone(), df_expr_to_unresolved(arg)?)?; - let mut agg_fn = agg_fn.clone(); - agg_fn.params.args[0] = Expr::Column(DfColumn::new_unqualified(alias)); - Ok(Expr::AggregateFunction(agg_fn)) - } - } - } - - /// Wrap `input` in the materializing `Project`, or return it untouched when - /// nothing needed deriving — so a query that lowers today keeps its exact - /// DAG, and a name collision that the projection would have flattened only - /// matters once the projection exists. - fn wrap(self, input: Unresolved) -> Result { - if !self.any { - return Ok(input); - } - if let Some(alias) = self.collision { - return Err(LoweringError::UnsupportedFeature(format!( - "ambiguous column `{alias}` beneath an expression GROUP BY / \ - aggregate — alias the relations apart" - ))); - } - Ok(Unresolved::Project { - cols: self.cols, - qualifier: None, - child: Rc::new(input), - }) - } -} - -/// The first aggregate argument's column name (bare / aliased / cast column), -/// or `None` for `*` / a non-column expression. -fn agg_col_name(args: &[Expr]) -> Option { - fn col_name(e: &Expr) -> Option { - match e { - Expr::Column(c) => Some(c.name.clone()), - Expr::Alias(a) => col_name(&a.expr), - Expr::Cast(c) => col_name(&c.expr), - _ => None, - } - } - args.first().and_then(col_name) -} - -/// The single input column of a value reducer (`SUM`/`MIN`/`MAX`/`AVG`/stddev/ -/// variance/quantile/count-distinct). Errors if the argument is not a column: -/// the canonical `AggIntent` reduces a column, not an arbitrary expression -/// (`SUM(a*b)`), so silently picking a probe column would compute the wrong -/// result. -fn reducer_col(name: &str, args: &[Expr]) -> Result { - agg_col_name(args).map(ColumnRef::Named).ok_or_else(|| { - LoweringError::UnsupportedAggregate(format!("{name} over a non-column expression")) - }) -} - -/// One argument of a `COUNT(DISTINCT ...)`. Resolved the way a grouping key is -/// — what is being counted is an identity, and its qualifier has to survive a -/// join (`a.k` vs `b.k`) — but reported as an aggregate restriction, since an -/// aggregate call is what the user wrote. -fn distinct_col(expr: &Expr) -> Result { - expr_to_group_ref(expr).map_err(|_| { - LoweringError::UnsupportedAggregate( - "COUNT(DISTINCT ...) over a non-column expression".into(), - ) - }) -} - -fn expr_to_group_ref(expr: &Expr) -> Result { - match expr { - // Preserve the relation qualifier so a GROUP BY / PARTITION BY key over a - // join (`b.k` vs `a.k`) resolves to the correct side — the same rule the - // scalar predicate path uses (`df_expr_to_unresolved`). - Expr::Column(col) => Ok(match &col.relation { - Some(rel) => ColumnRef::Qualified { - table: rel.to_string(), - name: col.name.clone(), - }, - None => ColumnRef::Named(col.name.clone()), - }), - Expr::Alias(a) => expr_to_group_ref(&a.expr), - other => Err(LoweringError::UnsupportedFeature(format!( - "non-column GROUP BY expression: {other}" - ))), - } -} - -fn extract_percentile_q(args: &[Expr]) -> Result { - let q = match args.get(1) { - Some(Expr::Literal(DfScalarValue::Float64(Some(q)), _)) => *q, - Some(Expr::Literal(DfScalarValue::Float32(Some(q)), _)) => *q as f64, - _ => { - return Err(LoweringError::InvalidExpression( - "percentile value must be a float literal (2nd arg)".into(), - )) - } - }; - if q.is_finite() && (0.0..=1.0).contains(&q) { - Ok(q) - } else { - Err(LoweringError::InvalidExpression(format!( - "percentile must be in [0, 1], got {q}" - ))) - } -} - -// ── LogicalPlan navigation helpers ────────────────────────────────────────────── - -fn eval_fetch(expr_opt: &Option>) -> Option { - expr_opt.as_ref().and_then(|e| match e.as_ref() { - Expr::Literal(DfScalarValue::Int64(Some(v)), _) if *v >= 0 => Some(*v as usize), - Expr::Literal(DfScalarValue::UInt64(Some(v)), _) => Some(*v as usize), - Expr::Literal(DfScalarValue::Int32(Some(v)), _) if *v >= 0 => Some(*v as usize), - _ => None, - }) -} - -/// Map a DataFusion window-function definition to the canonical -/// [`WindowFuncKind`]. -/// `NthValue` is returned with `None`; `lower_window` fills in `n` from args. -fn lower_window_func_kind(fun: &WindowFunctionDefinition) -> Result { - let unsupported = |what: &str, name: &str| { - LoweringError::UnsupportedFeature(format!("window {what}: {name}")) - }; - match fun { - WindowFunctionDefinition::WindowUDF(udf) => match udf.name().to_lowercase().as_str() { - "row_number" => Ok(WindowFuncKind::RowNumber), - "rank" => Ok(WindowFuncKind::Rank), - "dense_rank" => Ok(WindowFuncKind::DenseRank), - "lag" => Ok(WindowFuncKind::Lag), - "lead" => Ok(WindowFuncKind::Lead), - // ClickHouse: frame-respecting variants, not plain Lag/Lead (#267). - "laginframe" => Ok(WindowFuncKind::LagInFrame), - "leadinframe" => Ok(WindowFuncKind::LeadInFrame), - "first_value" => Ok(WindowFuncKind::FirstValue), - "last_value" => Ok(WindowFuncKind::LastValue), - "nth_value" => Ok(WindowFuncKind::NthValue(None)), - other => Err(unsupported("function", other)), - }, - WindowFunctionDefinition::AggregateUDF(udf) => match udf.name().to_lowercase().as_str() { - "sum" => Ok(WindowFuncKind::Sum), - "avg" | "mean" => Ok(WindowFuncKind::Avg), - "count" => Ok(WindowFuncKind::Count), - "min" => Ok(WindowFuncKind::Min), - "max" => Ok(WindowFuncKind::Max), - other => Err(unsupported("aggregate", other)), - }, - } -} - -/// Map DataFusion's resolved `WindowFrame` (issue #268) to the canonical -/// [`WindowFrame`]. DataFusion's planner always fills in the SQL-standard -/// default frame before the logical plan is built, so this never sees an -/// "absent" frame — only `ROWS`/`RANGE`/`GROUPS` with concrete bounds. -/// `GROUPS` is rejected: no query in this repo's SQL corpora uses it, and -/// nothing downstream interprets frame semantics yet, so it isn't worth -/// modelling untested. -fn lower_window_frame( - frame: &datafusion::logical_expr::WindowFrame, -) -> Result { - let units = match frame.units { - DfWindowFrameUnits::Rows => WindowFrameUnits::Rows, - DfWindowFrameUnits::Range => WindowFrameUnits::Range, - DfWindowFrameUnits::Groups => { - return Err(LoweringError::UnsupportedFeature( - "window frame unit: GROUPS".into(), - )) - } - }; - let offset = |v: &DfScalarValue| -> Result { - Ok(match v { - DfScalarValue::IntervalYearMonth(Some(months)) => WindowFrameOffset::Interval { - months: *months, - days: 0, - nanoseconds: 0, - }, - DfScalarValue::IntervalDayTime(Some(value)) => WindowFrameOffset::Interval { - months: 0, - days: value.days, - nanoseconds: i64::from(value.milliseconds) * 1_000_000, - }, - DfScalarValue::IntervalMonthDayNano(Some(value)) => WindowFrameOffset::Interval { - months: value.months, - days: value.days, - nanoseconds: value.nanoseconds, - }, - // DataFusion 43 keeps every RANGE offset as text: both numeric - // bounds such as `1.5` and normalized interval literals such as - // `"1 HOUR"`. Arrow's interval parser accepts bare numbers and - // interprets them as months, so classify numeric text first. - DfScalarValue::Utf8(Some(value)) | DfScalarValue::LargeUtf8(Some(value)) => { - if let Ok(value) = value.parse::() { - WindowFrameOffset::Scalar(ScalarValue::Int64(value)) - } else if let Ok(value) = value.parse::() { - WindowFrameOffset::Scalar(ScalarValue::Float64(value)) - } else { - match parse_interval_month_day_nano(value) { - Ok(interval) => WindowFrameOffset::Interval { - months: interval.months, - days: interval.days, - nanoseconds: interval.nanoseconds, - }, - Err(_) => WindowFrameOffset::Scalar(scalar_value_to_asap(v)?), - } - } - } - _ => WindowFrameOffset::Scalar(scalar_value_to_asap(v)?), - }) - }; - let bound = |b: &DfWindowFrameBound| -> Result { - Ok(match b { - DfWindowFrameBound::Preceding(v) => WindowFrameBound::Preceding(offset(v)?), - DfWindowFrameBound::CurrentRow => WindowFrameBound::CurrentRow, - DfWindowFrameBound::Following(v) => WindowFrameBound::Following(offset(v)?), - }) - }; - Ok(WindowFrame { - units, - start_bound: bound(&frame.start_bound)?, - end_bound: bound(&frame.end_bound)?, - }) -} - -// ── Issue #225, item 3: DataFusion registry drift detection ──────────────── -// -// `asap_sql_function_catalog::NATIVE_FUNCTIONS` is hand-maintained data -// mirroring what DataFusion's own aggregate-function registry resolves. That -// mirror can only silently drift out of sync — a DataFusion version bump -// that adds, renames, or removes a builtin aggregate leaves the catalog -// looking fine while `lower_agg_intent` quietly gains or loses coverage. Of -// the two introspectable sources the issue names, DataFusion's own registry -// is the one with no external dependency: `SessionContext` already lists its -// aggregate UDFs in-process, so the check below builds a real context the -// same way `build_context` does and walks it directly — no live database, -// no new CI infra, just `cargo test`. (ClickHouse's `system.functions` is -// the other source; it needs a live ClickHouse instance, which is handled -// separately by the dev-only `tools/clickhouse/extract_functions.py` script, -// deliberately not wired into this test or into CI.) -#[cfg(test)] -mod catalog_drift { - use super::*; - - /// Every aggregate function name DataFusion's planner resolves inside a - /// context built the same way `build_context` builds one must be - /// *accounted for* by the catalog: either `lookup_native` maps it to a - /// canonical semantic, it is one of our own `CLICKHOUSE_BUILTINS` stub - /// registrations (`build_context` registers those into the very same - /// context, so they show up here too), or it is explicitly listed in - /// `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` with a reason. - /// - /// This does *not* assert the reverse (that every `NATIVE_FUNCTIONS` - /// entry is resolvable) — a name that stops resolving after a DataFusion - /// bump just becomes permanently unreachable dead data, not a lowering - /// hazard, so it's out of scope for a regression gate. It also does not - /// try to derive `AggSemantic` from anything DataFusion reports — that - /// judgment call stays with whoever adds the catalog entry. - #[test] - fn every_datafusion_aggregate_name_is_covered_by_the_catalog() { - let catalog = SqlCatalog::new(); - let ctx = SqlLowerer::new(&catalog) - .build_context() - .expect("build_context with an empty table catalog cannot fail"); - let state = ctx.state(); - let mut uncovered: Vec<&str> = state - .aggregate_functions() - .keys() - .map(String::as_str) - .filter(|name| { - asap_sql_function_catalog::lookup_native(name).is_none() - && asap_sql_function_catalog::lookup_clickhouse_builtin(name).is_none() - && !asap_sql_function_catalog::KNOWN_UNMAPPED_NATIVE_FUNCTIONS.contains(name) - }) - .collect(); - uncovered.sort_unstable(); - assert!( - uncovered.is_empty(), - "DataFusion resolves these aggregate names but the catalog doesn't know about them \ - (crates/sql-function-catalog/src/lib.rs): {uncovered:?}\n\ - Either add a `NativeFunction` entry mapping each to its `AggSemantic`, or -- if it's \ - a deliberate non-goal (no `AggIntent` shape for it, or it's rejected elsewhere) -- \ - add it to `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` with a reason. This usually means a \ - DataFusion version bump added or renamed a builtin aggregate." - ); - } - - /// Every `KNOWN_UNMAPPED_NATIVE_FUNCTIONS` entry earns its place by - /// actually being a name DataFusion resolves today — otherwise it is - /// stale documentation for a name that no longer exists (e.g. a prior - /// DataFusion version renamed it), not a real "deliberately not mapped" - /// decision, and should be removed. - #[test] - fn known_unmapped_entries_are_all_real_datafusion_names() { - let catalog = SqlCatalog::new(); - let ctx = SqlLowerer::new(&catalog) - .build_context() - .expect("build_context with an empty table catalog cannot fail"); - let resolved = ctx.state().aggregate_functions().clone(); - for name in asap_sql_function_catalog::KNOWN_UNMAPPED_NATIVE_FUNCTIONS { - assert!( - resolved.contains_key(*name), - "`{name}` is listed in KNOWN_UNMAPPED_NATIVE_FUNCTIONS but DataFusion no longer \ - resolves it -- remove the stale entry" - ); - } - } -} diff --git a/crates/frontend-sql/src/sql/types.rs b/crates/frontend-sql/src/sql/types.rs deleted file mode 100644 index 1d178daf4..000000000 --- a/crates/frontend-sql/src/sql/types.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! Type bridges between DataFusion's Arrow types and the canonical `DataType`, plus -//! the SQL table catalog used to register tables with DataFusion and to carry -//! resolved leaf schemas into the canonical, unresolved DAG. - -use std::collections::HashMap; - -use datafusion::arrow::datatypes::{ - DataType as ArrowDataType, Field as ArrowField, Fields, Schema as ArrowSchema, -}; -use datafusion::common::ScalarValue as DfScalarValue; - -use asap_types::pre_asap::schema::{DataType, Field, Schema}; -use asap_types::pre_asap::ScalarValue; - -use crate::error::SqlError as LoweringError; - -/// Table catalog for SQL lowering: table name → resolved canonical [`Schema`]. -/// -/// Used twice: to register Arrow-backed `MemTable`s so DataFusion can resolve -/// `SELECT … FROM t`, and to attach each table's schema directly onto the -/// canonical `Scan` (`schema: Some(_)`) so the SchemaResolver doesn't need to -/// usage-derive it. -#[derive(Debug, Clone, Default)] -pub struct SqlCatalog { - pub tables: HashMap, -} - -impl SqlCatalog { - pub fn new() -> Self { - Self::default() - } - - /// Builder: register `name` with its resolved canonical schema. - pub fn with_table(mut self, name: impl Into, schema: Schema) -> Self { - self.tables.insert(name.into(), schema); - self - } -} - -pub(super) fn scalar_value_to_asap(sv: &DfScalarValue) -> Result { - match sv { - DfScalarValue::Int64(Some(v)) => Ok(ScalarValue::Int64(*v)), - DfScalarValue::Int32(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::Int16(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::Int8(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::UInt64(Some(v)) => i64::try_from(*v).map(ScalarValue::Int64).map_err(|_| { - LoweringError::InvalidExpression(format!("UInt64 value {v} overflows i64")) - }), - DfScalarValue::UInt32(Some(v)) => Ok(ScalarValue::Int64(*v as i64)), - DfScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(*v)), - DfScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float64(*v as f64)), - DfScalarValue::Utf8(Some(s)) | DfScalarValue::LargeUtf8(Some(s)) => { - Ok(ScalarValue::Utf8(s.clone())) - } - DfScalarValue::Boolean(Some(b)) => Ok(ScalarValue::Boolean(*b)), - // All three of DataFusion's interval scalars land on one canonical - // shape; the narrower two simply leave the fields they do not carry - // at zero. - DfScalarValue::IntervalYearMonth(Some(months)) => Ok(ScalarValue::Interval { - months: *months, - days: 0, - nanos: 0, - }), - DfScalarValue::IntervalDayTime(Some(v)) => Ok(ScalarValue::Interval { - months: 0, - days: v.days, - nanos: i64::from(v.milliseconds) * 1_000_000, - }), - DfScalarValue::IntervalMonthDayNano(Some(v)) => Ok(ScalarValue::Interval { - months: v.months, - days: v.days, - nanos: v.nanoseconds, - }), - _ if sv.is_null() => Ok(ScalarValue::Null), - _ => Err(LoweringError::InvalidExpression(format!( - "unsupported scalar: {sv:?}" - ))), - } -} - -/// Arrow → the canonical `DataType` (used for `CAST` targets). Deliberately narrow. -pub(super) fn arrow_to_dtype(dt: &ArrowDataType) -> Result { - match dt { - ArrowDataType::Null => Ok(DataType::Null), - ArrowDataType::Int64 - | ArrowDataType::Int32 - | ArrowDataType::Int16 - | ArrowDataType::Int8 => Ok(DataType::Int64), - ArrowDataType::Float64 | ArrowDataType::Float32 => Ok(DataType::Float64), - ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => Ok(DataType::Utf8), - ArrowDataType::Boolean => Ok(DataType::Bool), - ArrowDataType::Timestamp(_, _) => Ok(DataType::Timestamp), - ArrowDataType::Date32 | ArrowDataType::Date64 => Ok(DataType::Date), - ArrowDataType::Interval(_) => Ok(DataType::Interval), - ArrowDataType::List(element) => Ok(DataType::List { - element: Box::new(Field::new( - element.name(), - arrow_to_dtype(element.data_type())?, - element.is_nullable(), - )), - }), - ArrowDataType::Struct(fields) => Ok(DataType::Struct { - fields: fields - .iter() - .map(|field| { - Ok(Field::new( - field.name(), - arrow_to_dtype(field.data_type())?, - field.is_nullable(), - )) - }) - .collect::, LoweringError>>()?, - }), - ArrowDataType::Map(entries, _) => { - let ArrowDataType::Struct(fields) = entries.data_type() else { - return Err(LoweringError::UnsupportedFeature( - "map entries must be a struct".into(), - )); - }; - if fields.len() != 2 || fields[0].is_nullable() { - return Err(LoweringError::UnsupportedFeature( - "map entries require a non-null key and a value".into(), - )); - } - Ok(DataType::Map { - key: Box::new(arrow_to_dtype(fields[0].data_type())?), - value: Box::new(arrow_to_dtype(fields[1].data_type())?), - value_nullable: fields[1].is_nullable(), - }) - } - other => Err(LoweringError::UnsupportedFeature(format!( - "Arrow type: {other:?}" - ))), - } -} - -/// The canonical `DataType` → Arrow (for registering catalog tables with DataFusion). -pub(super) fn dtype_to_arrow(dt: &DataType) -> ArrowDataType { - match dt { - DataType::Null => ArrowDataType::Null, - DataType::Int64 => ArrowDataType::Int64, - DataType::Float64 => ArrowDataType::Float64, - DataType::Utf8 => ArrowDataType::Utf8, - DataType::Bool => ArrowDataType::Boolean, - DataType::List { element } => ArrowDataType::List(std::sync::Arc::new(ArrowField::new( - &element.name, - dtype_to_arrow(&element.dtype), - element.nullable, - ))), - DataType::Struct { fields } => ArrowDataType::Struct( - fields - .iter() - .map(|field| { - ArrowField::new(&field.name, dtype_to_arrow(&field.dtype), field.nullable) - }) - .collect::>() - .into(), - ), - DataType::Map { - key, - value, - value_nullable, - } => ArrowDataType::Map( - std::sync::Arc::new(ArrowField::new( - "entries", - ArrowDataType::Struct( - vec![ - ArrowField::new("key", dtype_to_arrow(key), false), - ArrowField::new("value", dtype_to_arrow(value), *value_nullable), - ] - .into(), - ), - false, - )), - false, - ), - DataType::Timestamp => { - ArrowDataType::Timestamp(datafusion::arrow::datatypes::TimeUnit::Millisecond, None) - } - // Deliberately narrowing: `Date64` lowers to `DataType::Date` and comes - // back as `Date32`. Both spell the same calendar date and nothing in - // the planner reads the width; a catalog that wants `Date64` back would - // need a second variant carrying no planning information. - DataType::Date => ArrowDataType::Date32, - // Only reachable through a hand-built schema: `Interval` types a - // literal, and no catalog declares a column with it. Mapped to the - // same three-field shape `ScalarValue::Interval` carries rather than - // left to panic. - DataType::Interval => { - ArrowDataType::Interval(datafusion::arrow::datatypes::IntervalUnit::MonthDayNano) - } - } -} - -/// Build an Arrow schema from a canonical [`Schema`] (column name + type + nullability). -pub(super) fn schema_to_arrow(schema: &Schema) -> ArrowSchema { - let fields: Fields = schema - .fields - .iter() - .map(|c: &Field| { - ArrowField::new(&c.name, dtype_to_arrow(c.expect_plain_dtype()), c.nullable) - }) - .collect(); - ArrowSchema::new(fields) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Both Arrow date widths bridge to the one canonical `Date`, and it - /// registers back as `Date32` — the documented narrowing. - #[test] - fn both_arrow_date_widths_bridge_to_date() { - assert_eq!( - arrow_to_dtype(&ArrowDataType::Date32).unwrap(), - DataType::Date - ); - assert_eq!( - arrow_to_dtype(&ArrowDataType::Date64).unwrap(), - DataType::Date - ); - assert_eq!(dtype_to_arrow(&DataType::Date), ArrowDataType::Date32); - } - - /// Every Arrow interval width shares the canonical calendar interval type. - #[test] - fn interval_types_round_trip_through_the_catalog_bridge() { - use datafusion::arrow::datatypes::IntervalUnit; - for unit in [ - IntervalUnit::YearMonth, - IntervalUnit::DayTime, - IntervalUnit::MonthDayNano, - ] { - assert_eq!( - arrow_to_dtype(&ArrowDataType::Interval(unit)).unwrap(), - DataType::Interval - ); - } - assert_eq!( - arrow_to_dtype(&dtype_to_arrow(&DataType::Interval)).unwrap(), - DataType::Interval - ); - } - - /// All three of DataFusion's interval scalars carry into the one canonical - /// three-field shape, with the fields they do not spell left at zero. - #[test] - fn every_datafusion_interval_scalar_carries_across() { - use datafusion::arrow::datatypes::{IntervalDayTime, IntervalMonthDayNano}; - - assert_eq!( - scalar_value_to_asap(&DfScalarValue::IntervalYearMonth(Some(14))).unwrap(), - ScalarValue::Interval { - months: 14, - days: 0, - nanos: 0 - } - ); - assert_eq!( - scalar_value_to_asap(&DfScalarValue::IntervalDayTime(Some(IntervalDayTime::new( - 30, 500 - )))) - .unwrap(), - ScalarValue::Interval { - months: 0, - days: 30, - nanos: 500_000_000 - } - ); - assert_eq!( - scalar_value_to_asap(&DfScalarValue::IntervalMonthDayNano(Some( - IntervalMonthDayNano::new(1, 2, 3) - ))) - .unwrap(), - ScalarValue::Interval { - months: 1, - days: 2, - nanos: 3 - } - ); - } - - /// Nested map values and value nullability survive catalog registration. - #[test] - fn nested_map_schema_round_trip() { - let map = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Map { - key: Box::new(DataType::Int64), - value: Box::new(DataType::Float64), - value_nullable: true, - }), - value_nullable: false, - }; - assert_eq!(arrow_to_dtype(&dtype_to_arrow(&map)).unwrap(), map); - let encoded = serde_json::to_string(&map).unwrap(); - assert_eq!(serde_json::from_str::(&encoded).unwrap(), map); - } -} - -#[cfg(test)] -mod collection_tests { - use super::*; - #[test] - fn nested_collections_preserve_field_names_order_and_nullability() { - let dtype = DataType::Struct { - fields: vec![ - Field::new( - "samples", - DataType::List { - element: Box::new(Field::new( - "sample", - DataType::Struct { - fields: vec![ - Field::new("timestamp", DataType::Timestamp, false), - Field::new("value", DataType::Float64, true), - Field::new( - "labels", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::List { - element: Box::new(Field::new( - "label_value", - DataType::Utf8, - false, - )), - }), - value_nullable: true, - }, - true, - ), - ], - }, - true, - )), - }, - false, - ), - Field::new("optional", DataType::Int64, true), - ], - }; - let arrow = dtype_to_arrow(&dtype); - assert_eq!(arrow_to_dtype(&arrow).unwrap(), dtype); - assert_eq!(dtype_to_arrow(&arrow_to_dtype(&arrow).unwrap()), arrow); - let encoded = serde_json::to_string(&dtype).unwrap(); - assert_eq!(serde_json::from_str::(&encoded).unwrap(), dtype); - } - #[test] - fn empty_struct_and_nonnullable_list_element_roundtrip() { - let dtype = DataType::List { - element: Box::new(Field::new( - "empty", - DataType::Struct { fields: vec![] }, - false, - )), - }; - assert_eq!(arrow_to_dtype(&dtype_to_arrow(&dtype)).unwrap(), dtype); - } -} - -#[cfg(test)] -mod bottom_map_tests { - use super::*; - #[test] - fn empty_map_bottom_types_roundtrip_without_string_defaults() { - let (map, nullable) = asap_types::pre_asap::scalar_type_rules::MapScalarFunction::Construct - .output_type(&[]) - .unwrap(); - assert!(!nullable); - let arrow = dtype_to_arrow(&map); - assert_eq!(arrow_to_dtype(&arrow).unwrap(), map); - assert_eq!( - arrow_to_dtype(&ArrowDataType::Null).unwrap(), - DataType::Null - ); - } -} diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs deleted file mode 100644 index bac54c54d..000000000 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ /dev/null @@ -1,2718 +0,0 @@ -//! End-to-end SQL → unresolved → canonical DAG lowering tests (positional IR). -//! -//! Validates the DataFusion front end: SQL parses + plans, lowers directly to -//! the canonical, unresolved shape (`QueryExpr`, issue #179), and -//! the shared `resolve_root` produces the positional, resolved canonical -//! DAG (the same resolver the PromQL path uses). - -use asap_frontend_sql::{lower_sql, lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; -use asap_types::pre_asap::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::pre_asap::{ - AggIntent, CompareOpKind, GroupKeys, JoinKind, Predicate, QueryExpr, Reduction, ScalarValue, - Source, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, WindowFuncKind, -}; -use asap_types::types::AccuracyTarget; -use asap_types::workload::SqlDialect; - -fn col(name: &str, dtype: DataType) -> Field { - Field::plain(name, dtype, false) -} - -/// `metrics(ts, service, latency, bytes)` + `hosts(service, region)`. -fn catalog() -> SqlCatalog { - SqlCatalog::new() - .with_table( - "metrics", - Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("latency", DataType::Float64), - col("bytes", DataType::Int64), - ], - 0, - vec![vec![0, 1]], - ), - ) - .with_table( - "hosts", - Schema::new(vec![ - col("service", DataType::Utf8), - col("region", DataType::Utf8), - ]), - ) -} - -async fn lower(sql: &str) -> QueryExpr { - lower_sql(sql, &catalog(), AccuracyTarget::Exact) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) -} - -#[tokio::test] -async fn planning_subquery_bridge_reuses_canonical_promql_subquery() { - let query = lower( - "SELECT max(value) FROM (\ - SELECT asap_promql_subquery(21600000, 60000) AS value FROM (\ - SELECT sum(bytes) AS value FROM metrics))", - ) - .await; - let QueryExpr::Project { child, .. } = query else { - panic!("expected outer SQL projection"); - }; - let QueryExpr::Aggregate { child, .. } = child.as_ref() else { - panic!("expected outer max aggregate, got {child:?}"); - }; - let QueryExpr::PromqlSubquery { - range, - resolution, - child, - } = child.as_ref() - else { - panic!("expected canonical subquery bridge, got {child:?}"); - }; - assert_eq!(*range, std::time::Duration::from_secs(6 * 60 * 60)); - assert_eq!(*resolution, Some(std::time::Duration::from_secs(60))); - assert!(matches!(child.as_ref(), QueryExpr::Project { .. })); -} - -#[tokio::test] -async fn planning_histogram_bridge_reuses_classic_bucket_intent() { - let query = lower( - "SELECT asap_histogram_quantile(0.95) AS value FROM (\ - SELECT service AS le, sum(bytes) AS value FROM metrics GROUP BY service)", - ) - .await; - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = query - else { - panic!("expected canonical histogram aggregate"); - }; - // One histogram over all rows; the bucket bound is the child's column 0. - assert!(reduction.expect_reduce().keys().is_empty()); - assert!(matches!( - measures.as_slice(), - [AggIntent::HistogramQuantile { q, le: 0 }] if (*q - 0.95).abs() < 1e-12 - )); - assert!(matches!(child.as_ref(), QueryExpr::Project { .. })); -} - -#[tokio::test] -async fn planning_relation_bridges_reject_ambiguous_shapes() { - let missing_alias = lower_sql( - "SELECT asap_promql_subquery(300000, 60000) FROM metrics", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .unwrap_err(); - assert!(missing_alias.to_string().contains("must have an alias")); - - let histogram_with_extra_column = lower_sql( - "SELECT service, asap_histogram_quantile(0.95) AS value FROM metrics", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .unwrap_err(); - assert!(histogram_with_extra_column - .to_string() - .contains("only expression")); - - let invalid_q = lower_sql( - "SELECT asap_histogram_quantile(1.5) AS value FROM metrics", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .unwrap_err(); - assert!(invalid_q.to_string().contains("finite and in [0,1]")); -} - -/// Find the first `Aggregate` node along the single-child spine. -fn find_aggregate(qe: &QueryExpr) -> Option<(&GroupKeys, &Vec)> { - match qe { - QueryExpr::Aggregate { - reduction, - measures, - .. - } => Some((reduction.expect_reduce(), measures)), - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => find_aggregate(child), - _ => None, - } -} - -/// The first `Aggregate` node itself, for tests that need its child. -fn find_aggregate_node(qe: &QueryExpr) -> Option<&QueryExpr> { - match qe { - QueryExpr::Aggregate { .. } => Some(qe), - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => find_aggregate_node(child), - _ => None, - } -} - -/// The names of the columns the first `Aggregate`'s reducers read, resolved -/// against its child's schema, plus whether that child is a materializing -/// `Project` (issue #110). -fn reducer_input_names(qe: &QueryExpr) -> (Vec, bool) { - let QueryExpr::Aggregate { - measures, child, .. - } = find_aggregate_node(qe).expect("expected an Aggregate") - else { - unreachable!() - }; - let schema = child.output_schema().expect("child schema"); - let names = measures - .iter() - .flat_map(|a| a.input_cols()) - .map(|id| schema.fields[id].name.clone()) - .collect(); - (names, matches!(**child, QueryExpr::Project { .. })) -} - -/// Find the first `Join` node along the single-child spine. -fn find_join(qe: &QueryExpr) -> Option<&QueryExpr> { - match qe { - QueryExpr::Join { .. } => Some(qe), - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => find_join(child), - _ => None, - } -} - -/// The first `Filter` node along the single-child spine. -fn find_filter(qe: &QueryExpr) -> Option<&QueryExpr> { - match qe { - QueryExpr::Filter { .. } => Some(qe), - QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => find_filter(child), - _ => None, - } -} - -#[tokio::test] -async fn where_folds_predicate_onto_scan() { - // WHERE folds onto the Scan predicates, below the SELECT projection. - let qe = lower("SELECT * FROM metrics WHERE service = 'api'").await; - let QueryExpr::Project { child, .. } = &qe else { - panic!("expected Project at root, got {qe:?}"); - }; - let QueryExpr::Scan { - source, - predicates, - schema, - } = child.as_ref() - else { - panic!("expected Scan under the projection, got {child:?}"); - }; - assert!(matches!(source, Source::Table { table_ref } if table_ref == "metrics")); - assert_eq!(predicates.len(), 1, "WHERE clause folded onto the scan"); - assert!( - schema.closed, - "a catalog-backed SQL scan has a closed schema" - ); -} - -#[tokio::test] -async fn multi_aggregate_group_by_binds_columns_positionally() { - // SUM(bytes)=col 3, AVG(latency)=col 2, GROUP BY service=col 1. - let qe = lower("SELECT service, SUM(bytes), AVG(latency) FROM metrics GROUP BY service").await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate in the DAG"); - assert_eq!(by, &vec![1], "GROUP BY service → column 1"); - assert!( - measures.contains(&AggIntent::Sum { col: Some(3) }), - "SUM(bytes) → Sum{{col:3}}, got {measures:?}" - ); - assert!( - measures.contains(&AggIntent::Avg { col: Some(2) }), - "AVG(latency) → Avg{{col:2}}, got {measures:?}" - ); -} - -#[tokio::test] -async fn projection_over_aggregate_resolves_output_types_via_output_names() { - // The enclosing Projection references the aggregates by DataFusion's - // generated names (e.g. "sum(metrics.bytes)"); output_names threads those - // onto the canonical Aggregate so the Project resolves real types — not - // the Utf8 fallback that an unresolved column would get. - let qe = lower("SELECT SUM(bytes), AVG(latency) FROM metrics").await; - let schema = qe - .output_schema() - .expect("root projection schema derivation"); - assert_eq!(schema.fields.len(), 2); - assert_eq!( - schema.fields[0].dtype, - DataType::Int64, - "SUM(bytes:Int64) resolves to Int64, not the Utf8 fallback" - ); - assert_eq!( - schema.fields[1].dtype, - DataType::Float64, - "AVG(latency) resolves to Float64" - ); -} - -#[tokio::test] -async fn single_agg_group_by_keeps_key_in_output_schema() { - // A tabular single-aggregate GROUP BY routes through the positional - // Aggregate.by path (not the PromQL fused-Partition shape), so the group - // key is a real output column the enclosing SELECT projection resolves. - let qe = lower("SELECT service, SUM(bytes) FROM metrics GROUP BY service").await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate (not a Partition)"); - assert_eq!(by, &vec![1], "GROUP BY service → Aggregate.by column 1"); - assert!(matches!( - measures.as_slice(), - [AggIntent::Sum { col: Some(3) }] - )); - - // Both the group key and the aggregate resolve in the root projection schema. - let schema = qe.output_schema().expect("root projection schema"); - assert_eq!(schema.fields.len(), 2); - assert_eq!( - schema.fields[0].dtype, - DataType::Utf8, - "service is in the output" - ); - assert_eq!(schema.fields[1].dtype, DataType::Int64, "SUM(bytes)"); -} - -#[tokio::test] -async fn count_ranked_topk_is_heavy_hitter() { - // `ORDER BY COUNT(*) DESC LIMIT k` over a single COUNT aggregate is the one - // case the heavy-hitter (frequency) sketch is correct for. The shared - // `canonicalize` pass (issue #34) promotes it to the canonical two-level - // form: an outer global `TopK` (by: []) over the explicit inner `Count` - // grouped by `service`. - let qe = lower( - "SELECT service, COUNT(*) FROM metrics GROUP BY service ORDER BY COUNT(*) DESC LIMIT 10", - ) - .await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - by.is_empty(), - "outer TopK is a global ranking (by: []), got {by:?}" - ); - assert!( - matches!(measures.as_slice(), [AggIntent::TopK { k: 10, .. }]), - "count-ranked topk → heavy-hitter TopK, got {measures:?}" - ); - // The inner child is the explicit Count, grouped by service (col 1). - let QueryExpr::Aggregate { child, .. } = &qe else { - panic!("expected outer Aggregate, got {qe:?}"); - }; - let (inner_by, inner_measures) = find_aggregate(child).expect("expected inner Count aggregate"); - assert_eq!(inner_by, &vec![1], "inner Count grouped by service → col 1"); - assert!( - matches!(inner_measures.as_slice(), [AggIntent::Count { .. }]), - "inner aggregate is the explicit Count, got {inner_measures:?}" - ); -} - -#[tokio::test] -async fn count_ranked_topk_via_alias_is_also_heavy_hitter() { - // Regression for #20: aliasing `COUNT(*)` in the ORDER BY used to defeat the - // SQL front-end gate. The positional `canonicalize` pass now promotes it too, - // so the aliased and inline forms produce an identical canonical DAG. - let inline = lower( - "SELECT service, COUNT(*) FROM metrics GROUP BY service ORDER BY COUNT(*) DESC LIMIT 10", - ) - .await; - let aliased = lower( - "SELECT service, COUNT(*) AS cnt FROM metrics GROUP BY service ORDER BY cnt DESC LIMIT 10", - ) - .await; - assert_eq!( - inline, aliased, - "aliased count-ranked topk must match the inline form" - ); - let (_, measures) = find_aggregate(&aliased).expect("expected an Aggregate"); - assert!( - matches!(measures.as_slice(), [AggIntent::TopK { k: 10, .. }]), - "aliased count-ranked topk → heavy-hitter TopK, got {measures:?}" - ); -} - -#[tokio::test] -async fn non_count_ranked_limit_keeps_the_aggregate() { - // Ranking by AVG (not a count) must NOT become a frequency heavy-hitter — - // the AVG aggregate has to survive as a generic Sort+Limit. - let qe = lower( - "SELECT service, AVG(latency) AS a FROM metrics GROUP BY service ORDER BY a DESC LIMIT 10", - ) - .await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - measures.iter().any(|a| matches!(a, AggIntent::Avg { .. })), - "AVG must be preserved, got {measures:?}" - ); - assert!( - !measures.iter().any(|a| matches!(a, AggIntent::TopK { .. })), - "AVG ranking must not become a frequency heavy-hitter, got {measures:?}" - ); -} - -#[tokio::test] -async fn distinct_value_reducer_is_rejected_not_dropped() { - // The canonical intent algebra has no distinct-Sum; SUM(DISTINCT x) must - // be rejected, not silently lowered as SUM(x). - let res = lower_sql( - "SELECT SUM(DISTINCT bytes) FROM metrics", - &catalog(), - AccuracyTarget::Exact, - ) - .await; - assert!(res.is_err(), "SUM(DISTINCT ...) should be rejected"); -} - -#[tokio::test] -async fn aggregate_over_an_expression_reduces_a_derived_column() { - // The canonical `AggIntent` reduces a column, not an arbitrary expression. - // `SUM(bytes + 1)` used to be rejected for that reason; since #110 the - // expression is materialized as a derived column in a `Project` beneath - // the aggregate, and reduced there. - let qe = lower("SELECT SUM(bytes + 1) FROM metrics").await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!(measures.as_slice(), [AggIntent::Sum { col: Some(_) }]), - "expected Sum bound to the derived column, got {measures:?}" - ); - let (names, materialized) = reducer_input_names(&qe); - assert!(materialized, "expected a materializing Project"); - assert!( - names[0].contains("bytes") && names[0].contains('1'), - "the reduced column should be the projected `bytes + 1`, got {names:?}" - ); -} - -#[tokio::test] -async fn count_star_is_count_intent() { - let qe = lower("SELECT COUNT(*) FROM metrics").await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!(by.is_empty()); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); -} - -#[tokio::test] -async fn count_distinct_is_cardinality() { - let qe = lower("SELECT COUNT(DISTINCT service) FROM metrics").await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!(matches!( - measures.as_slice(), - [AggIntent::Cardinality { .. }] - )); -} - -#[tokio::test] -async fn select_distinct_lowers_to_distinct_with_positional_cols() { - // SELECT DISTINCT → a `Dedup` node whose `cols` are positional ColumnIds - // (not name-based ColumnRefs). DataFusion's `Distinct::All` dedups on every - // column, so `cols` is empty here — but the field type is now `Vec`. - let qe = lower("SELECT DISTINCT service FROM metrics").await; - let QueryExpr::Dedup { cols, .. } = &qe else { - panic!("expected a Dedup at the root, got {qe:?}"); - }; - let _: &Vec = cols; // compile-time: positional ids, not ColumnRefs - assert!(cols.is_empty(), "DISTINCT * dedups on all columns"); -} - -#[tokio::test] -async fn inner_join_lowers_to_join_over_two_scans() { - // INNER JOIN over two distinct tables → a canonical Join with both leaves as Scans. - let qe = lower( - "SELECT metrics.bytes, hosts.region \ - FROM metrics JOIN hosts ON metrics.service = hosts.service", - ) - .await; - let join = find_join(&qe).expect("expected a Join in the DAG"); - let QueryExpr::Join { - kind, left, right, .. - } = join - else { - unreachable!("find_join only returns Join"); - }; - assert_eq!(*kind, JoinKind::Inner); - assert!(matches!(left.as_ref(), QueryExpr::Scan { .. })); - assert!(matches!(right.as_ref(), QueryExpr::Scan { .. })); -} - -/// The two `ColumnId`s an equijoin predicate `Column(l) = Column(r)` binds to, -/// returned sorted so the assertion is independent of left/right ordering. -fn join_eq_columns(join: &QueryExpr) -> [usize; 2] { - let QueryExpr::Join { pred, .. } = join else { - unreachable!("expected a Join"); - }; - let QueryExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - } = pred.0.as_ref() - else { - panic!("expected an equijoin Compare, got {:?}", pred.0); - }; - match (left.as_ref(), right.as_ref()) { - (QueryExpr::Column(l), QueryExpr::Column(r)) => { - let mut cols = [*l, *r]; - cols.sort_unstable(); - cols - } - other => panic!("expected Field = Field, got {other:?}"), - } -} - -#[tokio::test] -async fn join_predicate_disambiguates_shared_column_name() { - // Issue #7: `metrics.service = hosts.service` shares a column name across the - // join. The qualified refs must bind to two *distinct* positions in the - // concatenated schema, not collapse onto the first `service`. - // metrics(ts,service,latency,bytes) ++ hosts(service,region) - // → metrics.service = col 1, hosts.service = col 4. - let qe = lower( - "SELECT metrics.bytes, hosts.region \ - FROM metrics JOIN hosts ON metrics.service = hosts.service", - ) - .await; - let join = find_join(&qe).expect("expected a Join in the DAG"); - assert_eq!( - join_eq_columns(join), - [1, 4], - "join key must bind to distinct positions, not the same `service`" - ); -} - -#[tokio::test] -async fn derived_table_join_disambiguates_via_alias() { - // Issue #66: a join over two *derived tables* must bind its keys to distinct - // positions. Before the fix the derived output columns lost their qualifier, - // so `a.service` and `b.service` both fell back to the first bare `service` - // (col 0) — `service = service`, always true → a silent cross product. - // Concatenated: a[service,region] ++ b[service,region] → a.service=0, b.service=2. - let qe = lower( - "SELECT a.region, b.region \ - FROM (SELECT service, region FROM hosts) a \ - JOIN (SELECT service, region FROM hosts) b ON a.service = b.service", - ) - .await; - let join = find_join(&qe).expect("expected a Join in the DAG"); - assert_eq!( - join_eq_columns(join), - [0, 2], - "derived-table join keys must bind to distinct positions, not both to the first `service`" - ); -} - -#[tokio::test] -async fn derived_table_select_star_join_disambiguates_via_alias() { - // Same as above but `SELECT *` derived tables (the non-Projection path that - // wraps the inner plan in an identity re-qualifying projection). - let qe = lower( - "SELECT a.region, b.region \ - FROM (SELECT * FROM hosts) a JOIN (SELECT * FROM hosts) b \ - ON a.service = b.service", - ) - .await; - let join = find_join(&qe).expect("expected a Join in the DAG"); - let [l, r] = join_eq_columns(join); - assert_ne!( - l, r, - "SELECT * derived-table join keys must not collapse to one column" - ); -} - -#[tokio::test] -async fn self_join_disambiguates_via_aliases() { - // A self-join shares *every* column name; the alias qualifiers (`a`/`b`) are - // the only way to tell the two `service` columns apart. - // metrics ++ metrics → a.service = col 1, b.service = col 5 (4 cols/side). - let qe = lower( - "SELECT a.bytes, b.latency \ - FROM metrics a JOIN metrics b ON a.service = b.service", - ) - .await; - let join = find_join(&qe).expect("expected a self-Join in the DAG"); - assert_eq!( - join_eq_columns(join), - [1, 5], - "self-join keys must bind to distinct sides" - ); -} - -#[tokio::test] -async fn qualified_where_over_join_resolves_to_right_side() { - // Issue #7 beyond the join key: a WHERE on the *duplicated* column name - // (`service` exists on both sides) must bind to the qualified side, not the - // first match. metrics.service = col 1, hosts.service = col 4 → `hosts.service` - // must resolve to 4. (Unoptimized plan keeps the Filter above the Join — no - // predicate pushdown — so it binds against the concatenated schema.) - let qe = lower( - "SELECT metrics.bytes FROM metrics JOIN hosts ON metrics.service = hosts.service \ - WHERE hosts.service = 'api'", - ) - .await; - let filter = find_filter(&qe).expect("expected a Filter over the join"); - let QueryExpr::Filter { pred, .. } = filter else { - unreachable!("find_filter only returns Filter"); - }; - assert!( - matches!(pred.0.as_ref(), QueryExpr::Compare { left, op: CompareOpKind::Eq, .. } - if matches!(left.as_ref(), QueryExpr::Column(4))), - "hosts.service must bind to concatenated position 4 (not the first `service`), got {:?}", - pred.0 - ); -} - -#[tokio::test] -async fn self_join_group_by_disambiguates_via_qualifier() { - // Group-key qualifier fix: GROUP BY on the *duplicated* column over a - // self-join must bind to the qualified side, not first-match. metrics ⋈ - // metrics → a.service = col 1, b.service = col 5. (Without qualified keys, - // both `GROUP BY a.service` and `GROUP BY b.service` collapsed to col 1.) - let qe_b = lower( - "SELECT b.service, COUNT(*) FROM metrics a JOIN metrics b \ - ON a.service = b.service GROUP BY b.service", - ) - .await; - let (by, _) = find_aggregate(&qe_b).expect("expected an Aggregate over the self-join"); - assert_eq!( - by, - &vec![5], - "GROUP BY b.service binds to the b side (col 5)" - ); - - let qe_a = lower( - "SELECT a.service, COUNT(*) FROM metrics a JOIN metrics b \ - ON a.service = b.service GROUP BY a.service", - ) - .await; - let (by, _) = find_aggregate(&qe_a).expect("expected an Aggregate over the self-join"); - assert_eq!( - by, - &vec![1], - "GROUP BY a.service binds to the a side (col 1)" - ); -} - -#[tokio::test] -async fn aggregate_over_join_binds_against_concatenated_schema() { - // GROUP BY a right-table column over a join: the key must resolve against - // the concatenated schema, exercising the bottom-up converter end to end. - // Two aggregates → the multi-agg path, which carries GROUP BY keys as - // positional `Aggregate.by` (as does every reducing GROUP BY). - let qe = lower( - "SELECT hosts.region, SUM(metrics.bytes), COUNT(*) \ - FROM metrics JOIN hosts ON metrics.service = hosts.service \ - GROUP BY hosts.region", - ) - .await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate over the join"); - // metrics(ts,service,latency,bytes) ++ hosts(service,region) → - // region is column 5, bytes is column 3 of the concatenated schema. - assert_eq!( - by, - &vec![5], - "GROUP BY hosts.region → concatenated column 5" - ); - assert!( - measures.contains(&AggIntent::Sum { col: Some(3) }), - "SUM(metrics.bytes) → Sum{{col:3}}, got {measures:?}" - ); -} - -// ── Issue #111: IN / EXISTS subquery predicates become semi / anti joins ──── - -/// The first `Join` node's `(kind, predicate, left column count)`. -fn join_parts(qe: &QueryExpr) -> (&JoinKind, &QueryExpr, usize) { - let QueryExpr::Join { - kind, - pred, - left, - right: _, - } = find_join(qe).expect("expected a Join") - else { - unreachable!() - }; - let left_len = left.output_schema().expect("left schema").fields.len(); - (kind, pred.0.as_ref(), left_len) -} - -#[tokio::test] -async fn in_subquery_lowers_to_a_semi_join() { - // `metrics(ts, service, latency, bytes)` — service is column 1. - let qe = - lower("SELECT service FROM metrics WHERE service IN (SELECT service FROM hosts)").await; - let (kind, pred, left_len) = join_parts(&qe); - assert_eq!(kind, &JoinKind::Semi); - assert_eq!(left_len, 4); - - // The predicate resolves against `left ++ right`. Both relations have a - // `service` column, so a name-based lookup would bind *both* sides to the - // left's — silently making this `service = service`, always true. The key is - // projected under a synthetic name to make that impossible. - let QueryExpr::Compare { left, right, .. } = pred else { - panic!("expected a comparison, got {pred:?}"); - }; - assert_eq!(**left, QueryExpr::Column(1), "outer service"); - assert_eq!( - **right, - QueryExpr::Column(left_len), - "the subquery key, not the outer column again" - ); -} - -#[tokio::test] -async fn a_semi_join_outputs_only_the_left_schema() { - // The right side is a filter, not a source of columns. - let qe = - lower("SELECT service FROM metrics WHERE service IN (SELECT service FROM hosts)").await; - let join = find_join(&qe).expect("expected a Join"); - let names: Vec<_> = join - .output_schema() - .expect("join schema") - .fields - .iter() - .map(|c| c.name.clone()) - .collect(); - assert_eq!(names, ["ts", "service", "latency", "bytes"]); -} - -#[tokio::test] -async fn a_subquery_key_that_is_an_expression_still_binds() { - // `SELECT bytes + 1 …` has no column name of its own; it is projected under - // the synthetic key rather than becoming an unreferenceable `col_0`. - let qe = - lower("SELECT service FROM metrics WHERE bytes IN (SELECT bytes + 1 FROM metrics)").await; - assert_eq!(join_parts(&qe).0, &JoinKind::Semi); -} - -#[tokio::test] -async fn a_multi_column_in_subquery_is_rejected() { - let err = lower_sql( - "SELECT service FROM metrics WHERE service IN (SELECT service, region FROM hosts)", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("IN must select one column"); - // DataFusion's planner rejects this before `lower_in_subquery`'s own - // arity check; either message names the one-column rule. - assert!(format!("{err}").contains("one column"), "got {err}"); -} - -#[tokio::test] -async fn an_ordinary_conjunct_still_folds_onto_the_scan() { - // The residual filter stays *below* the semi-join, where the converter can - // still fold it onto the Scan. A semi-join only drops left rows, so the - // orders agree. - let qe = lower( - "SELECT service FROM metrics WHERE bytes > 10 \ - AND service IN (SELECT service FROM hosts)", - ) - .await; - fn scan_has_predicate(qe: &QueryExpr) -> bool { - match qe { - QueryExpr::Scan { predicates, .. } => !predicates.is_empty(), - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Aggregate { child, .. } => scan_has_predicate(child), - QueryExpr::Join { left, right, .. } => { - scan_has_predicate(left) || scan_has_predicate(right) - } - _ => false, - } - } - assert_eq!(join_parts(&qe).0, &JoinKind::Semi); - assert!( - scan_has_predicate(&qe), - "WHERE bytes > 10 should reach the Scan" - ); -} - -/// Find the first `SQLWindowFunc` node along the single-child spine. -fn find_windowfunc(qe: &QueryExpr) -> Option<&QueryExpr> { - match qe { - QueryExpr::SQLWindowFunc { .. } => Some(qe), - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => find_windowfunc(child), - _ => None, - } -} - -#[tokio::test] -async fn window_function_lowers_to_positional_windowfunc() { - // ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC). - let qe = lower( - "SELECT service, ROW_NUMBER() OVER (PARTITION BY service ORDER BY bytes DESC) \ - FROM metrics", - ) - .await; - let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); - let QueryExpr::SQLWindowFunc { - func, - partition_by, - order_by, - .. - } = win - else { - unreachable!("find_windowfunc only returns SQLWindowFunc"); - }; - assert_eq!(*func, WindowFuncKind::RowNumber); - assert_eq!(partition_by, &vec![1], "PARTITION BY service → col 1"); - assert_eq!(order_by.len(), 1); - assert_eq!( - order_by[0].expr, - QueryExpr::Column(3), - "ORDER BY bytes → col 3" - ); - assert!(!order_by[0].ascending, "DESC"); - - // The window output column is appended to the schema (Int64 for ROW_NUMBER), - // and the enclosing projection resolves it (output_name threading). - let schema = qe.output_schema().expect("root schema"); - assert!( - schema.fields.iter().any(|c| c.dtype == DataType::Int64), - "row_number output column present, got {:?}", - schema.fields - ); -} - -#[tokio::test] -async fn window_aggregate_lowers_to_windowfunc() { - let qe = lower("SELECT service, SUM(bytes) OVER (PARTITION BY service) FROM metrics").await; - let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); - let QueryExpr::SQLWindowFunc { func, args, .. } = win else { - unreachable!(); - }; - assert_eq!(*func, WindowFuncKind::Sum); - assert_eq!(args, &vec![QueryExpr::Column(3)], "SUM(bytes) → arg col 3"); -} - -// ── Window frames (issue #268) ─────────────────────────────────────────────── - -/// The frame clause must actually reach the IR, not just the display string: -/// three window frames that differ semantically must lower to different -/// `SQLWindowFunc.frame` values. -#[tokio::test] -async fn window_frame_is_captured_not_dropped() { - let default_frame = - lower("SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts) FROM metrics") - .await; - let two_preceding = lower( - "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ - ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", - ) - .await; - let unbounded_following = lower( - "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ - ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM metrics", - ) - .await; - - let frame_of = |qe: &QueryExpr| { - let QueryExpr::SQLWindowFunc { frame, .. } = find_windowfunc(qe).unwrap() else { - unreachable!(); - }; - frame - .clone() - .expect("newly lowered SQL always records a frame") - }; - let (a, b, c) = ( - frame_of(&default_frame), - frame_of(&two_preceding), - frame_of(&unbounded_following), - ); - assert_ne!(a, b, "default frame vs ROWS 2 PRECEDING must differ"); - assert_ne!( - a, c, - "default frame vs ROWS CURRENT..UNBOUNDED FOLLOWING must differ" - ); - assert_ne!(b, c); - - assert_eq!(b.units, WindowFrameUnits::Rows); - assert_eq!( - b.start_bound, - WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Int64(2))) - ); - assert_eq!(b.end_bound, WindowFrameBound::CurrentRow); - - assert_eq!(c.start_bound, WindowFrameBound::CurrentRow); - assert_eq!( - c.end_bound, - WindowFrameBound::Following(WindowFrameOffset::Scalar(ScalarValue::Null)) - ); -} - -#[tokio::test] -async fn range_interval_frame_is_preserved() { - let qe = lower( - "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ - RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW) FROM metrics", - ) - .await; - let QueryExpr::SQLWindowFunc { - frame: Some(frame), .. - } = find_windowfunc(&qe).unwrap() - else { - panic!("expected a window function with a concrete frame"); - }; - - assert_eq!(frame.units, WindowFrameUnits::Range); - assert_eq!( - frame.start_bound, - WindowFrameBound::Preceding(WindowFrameOffset::Interval { - months: 0, - days: 0, - nanoseconds: 3_600_000_000_000, - }) - ); - assert_eq!(frame.end_bound, WindowFrameBound::CurrentRow); -} - -#[tokio::test] -async fn range_numeric_frames_remain_scalar_offsets() { - let integer = lower( - "SELECT SUM(bytes) OVER (ORDER BY bytes \ - RANGE BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", - ) - .await; - let fractional = lower( - "SELECT SUM(latency) OVER (ORDER BY latency \ - RANGE BETWEEN 1.5 PRECEDING AND CURRENT ROW) FROM metrics", - ) - .await; - - let start_bound = |qe: &QueryExpr| { - let QueryExpr::SQLWindowFunc { - frame: Some(frame), .. - } = find_windowfunc(qe).unwrap() - else { - panic!("expected a window function with a concrete frame"); - }; - frame.start_bound.clone() - }; - - assert_eq!( - start_bound(&integer), - WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Int64(2))) - ); - assert_eq!( - start_bound(&fractional), - WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Float64(1.5))) - ); -} - -/// `GROUPS` frames aren't in this repo's SQL corpora and nothing downstream -/// interprets frame semantics yet — rejected explicitly rather than silently -/// mis-lowered. -#[tokio::test] -async fn groups_frame_is_rejected() { - let err = lower_sql( - "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ - GROUPS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("GROUPS frame unit must be rejected"); - assert!(format!("{err}").contains("GROUPS"), "got {err}"); -} - -// ── Nested query functions: derived tables / inline views (issue #27) ─────────── - -/// Collect every `AggIntent` in the DAG, root-to-leaf. -fn all_intents(qe: &QueryExpr) -> Vec { - let mut out = Vec::new(); - fn go(qe: &QueryExpr, out: &mut Vec) { - match qe { - QueryExpr::Aggregate { - measures, child, .. - } => { - out.extend(measures.iter().cloned()); - go(child, out); - } - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::SQLWindowFunc { child, .. } - | QueryExpr::PromqlSubquery { child, .. } => go(child, out), - QueryExpr::BinaryOp { lhs, rhs, .. } - | QueryExpr::Join { - left: lhs, - right: rhs, - .. - } - | QueryExpr::SetOp { - left: lhs, - right: rhs, - .. - } => { - go(lhs, out); - go(rhs, out); - } - _ => {} - } - } - go(qe, &mut out); - out -} - -#[tokio::test] -async fn derived_table_aggregate_over_aggregate_nests() { - // `MAX(s)` over a derived table `(SELECT service, SUM(bytes) AS s … GROUP BY - // service)` — the SQL counterpart of PromQL function nesting (issue #27). - // Both reductions survive into the canonical DAG: an outer `Max` over - // the inner `Sum`. - let qe = lower( - "SELECT MAX(s) FROM \ - (SELECT service, SUM(bytes) AS s FROM metrics GROUP BY service) t", - ) - .await; - let intents = all_intents(&qe); - assert!( - intents.iter().any(|i| matches!(i, AggIntent::Max { .. })), - "outer MAX survives, got {intents:?}" - ); - assert!( - intents.iter().any(|i| matches!(i, AggIntent::Sum { .. })), - "inner SUM survives, got {intents:?}" - ); - // The whole nested DAG's output schema derives without error (positional - // resolution is total across the derived-table boundary). - assert_eq!(qe.output_schema().unwrap().fields.len(), 1); -} - -#[tokio::test] -async fn derived_table_outer_avg_over_inner_percentile() { - // Outer exact `AVG` over an inner approximate `Quantile` — each layer keeps - // its own intent (the per-node sketch-vs-exact choice is a post-ASAP decision). - let qe = lower( - "SELECT AVG(p) FROM \ - (SELECT service, approx_percentile_cont(latency, 0.9) AS p \ - FROM metrics GROUP BY service) t", - ) - .await; - let intents = all_intents(&qe); - assert!(intents.iter().any(|i| matches!(i, AggIntent::Avg { .. }))); - assert!(intents - .iter() - .any(|i| matches!(i, AggIntent::Quantile { q, .. } if (*q - 0.9).abs() < 1e-9))); -} - -#[tokio::test] -async fn filter_over_derived_aggregate_resolves_alias_column() { - // `WHERE t.s > 100` over a derived aggregate — the qualified ref `t.s` - // resolves by bare name against the derived output schema, and the Filter - // sits above the inner Aggregate. - let qe = lower( - "SELECT t.service, t.s FROM \ - (SELECT service, SUM(bytes) AS s FROM metrics GROUP BY service) t \ - WHERE t.s > 100", - ) - .await; - assert!( - find_filter(&qe).is_some(), - "the outer WHERE lowers to a Filter, got {qe:?}" - ); - assert!(all_intents(&qe) - .iter() - .any(|i| matches!(i, AggIntent::Sum { .. }))); - // Schema derivation is total across the boundary. - let _ = qe.output_schema().expect("nested schema derivation"); -} - -#[tokio::test] -async fn scalar_subquery_in_predicate_is_rejected() { - // A subquery-*valued* expression (`x > (SELECT …)`) needs a subquery node in - // the unresolved expression IR (and a correlated/uncorrelated decision); - // rejected cleanly until that lands. Derived tables in FROM (the common nesting - // shape) ARE supported — see the tests above. - let res = lower_sql( - "SELECT service FROM metrics WHERE bytes > (SELECT AVG(bytes) FROM metrics)", - &catalog(), - AccuracyTarget::Exact, - ) - .await; - assert!( - res.is_err(), - "scalar subquery in predicate should be rejected" - ); -} - -#[tokio::test] -async fn correlated_exists_lifts_its_correlation_into_the_join() { - // `EXISTS (SELECT 1 FROM hosts h WHERE h.service = m.service)` → a semi-join - // on `h.service = m.service`. The `SELECT 1` projection is dropped: a - // semi-join keeps no right columns, and it would have projected away the - // very column the correlation needs. - let qe = lower( - "SELECT service FROM metrics m WHERE EXISTS \ - (SELECT 1 FROM hosts h WHERE h.service = m.service)", - ) - .await; - let (kind, pred, left_len) = join_parts(&qe); - assert_eq!(kind, &JoinKind::Semi); - let QueryExpr::Compare { left, right, .. } = pred else { - panic!("expected the correlation as a comparison, got {pred:?}"); - }; - assert_eq!( - **left, - QueryExpr::Column(left_len), - "h.service (right side)" - ); - assert_eq!(**right, QueryExpr::Column(1), "m.service (left side)"); -} - -#[tokio::test] -async fn not_exists_lowers_to_an_anti_join() { - let qe = lower( - "SELECT service FROM metrics m WHERE NOT EXISTS \ - (SELECT 1 FROM hosts h WHERE h.service = m.service)", - ) - .await; - assert_eq!(join_parts(&qe).0, &JoinKind::Anti); -} - -#[tokio::test] -async fn an_uncorrelated_exists_is_an_unconditional_semi_join() { - // No correlation → keep every left row iff the right side has any row. - let qe = lower("SELECT service FROM metrics WHERE EXISTS (SELECT 1 FROM hosts)").await; - let (kind, pred, _) = join_parts(&qe); - assert_eq!(kind, &JoinKind::Semi); - assert_eq!(*pred, QueryExpr::Literal(ScalarValue::Boolean(true))); -} - -#[tokio::test] -async fn not_in_subquery_is_rejected_rather_than_mislowered_as_an_anti_join() { - // `NOT IN` is *not* an anti-join. Under three-valued logic a single NULL - // among the subquery's rows makes `c NOT IN (…)` UNKNOWN for every `c`, so - // the query returns nothing — while an anti-join returns every unmatched - // left row. Rejecting is the only correct option until the nullability is - // proven, and `NOT EXISTS` is the safe spelling. - let err = lower_sql( - "SELECT service FROM metrics WHERE service NOT IN (SELECT service FROM hosts)", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("NOT IN must not lower to an anti-join"); - assert!(format!("{err}").contains("NOT IN"), "got {err}"); -} - -#[tokio::test] -async fn a_correlated_in_subquery_is_rejected() { - let err = lower_sql( - "SELECT service FROM metrics m WHERE service IN \ - (SELECT h.service FROM hosts h WHERE h.region = m.service)", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("correlated IN needs both a key match and a correlation"); - assert!(format!("{err}").contains("correlated IN"), "got {err}"); -} - -// ── Issue #115: Quantile / Cardinality carry their input column ───────────── - -#[tokio::test] -async fn quantile_carries_its_input_column() { - // `metrics(ts=0, service=1, latency=2, bytes=3)`. Two quantiles over - // different columns must not compare equal — a workload-level dedupe pass - // would compare on `AggIntent` equality, so a col-less intent would - // collapse them. - let qe = lower( - "SELECT approx_percentile_cont(latency, 0.5), \ - approx_percentile_cont(bytes, 0.5) FROM metrics", - ) - .await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!( - measures.as_slice(), - [ - AggIntent::Quantile { col: Some(2), .. }, - AggIntent::Quantile { col: Some(3), .. } - ] - ), - "quantiles must bind their own column, got {measures:?}" - ); - assert_ne!( - measures[0], measures[1], - "distinct-column quantiles must not compare equal" - ); -} - -#[tokio::test] -async fn count_distinct_carries_its_input_column() { - let qe = lower("SELECT COUNT(DISTINCT service), COUNT(DISTINCT bytes) FROM metrics").await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!( - measures.as_slice(), - [ - AggIntent::Cardinality { cols: c1, .. }, - AggIntent::Cardinality { cols: c2, .. } - ] if c1 == &[1] && c2 == &[3] - ), - "cardinalities must bind their own column, got {measures:?}" - ); - assert_ne!( - measures[0], measures[1], - "distinct-column cardinalities must not compare equal" - ); -} - -#[tokio::test] -async fn quantile_and_count_distinct_over_an_expression_bind_the_derived_column() { - // A SQL aggregate has no "sample value" to fall back on, so an expression - // argument must never reach the canonical DAG as `col: None` (#115). - // Since #110 it reaches the canonical DAG as `col: Some(derived)` - // instead of being rejected. - for q in [ - "SELECT approx_percentile_cont(bytes * 8, 0.95) FROM metrics", - "SELECT COUNT(DISTINCT bytes * 8) FROM metrics", - "SELECT approx_distinct(bytes * 8) FROM metrics", - ] { - let qe = lower(q).await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - !measures[0].input_cols().is_empty(), - "{q} must bind a column, never the implicit input, got {measures:?}" - ); - let (names, materialized) = reducer_input_names(&qe); - assert!(materialized, "{q} expected a materializing Project"); - assert!( - names[0].contains("bytes"), - "{q} should reduce the projected `bytes * 8`, got {names:?}" - ); - } -} - -// ── Issue #111: median / approx_median → the φ=0.5 quantile ───────────────── - -#[tokio::test] -async fn median_lowers_to_the_half_quantile() { - // `metrics(ts=0, service=1, latency=2, bytes=3)`. - for sql in [ - "SELECT median(latency) FROM metrics", - "SELECT approx_median(latency) FROM metrics", - ] { - let qe = lower(sql).await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!( - measures.as_slice(), - [AggIntent::Quantile { col: Some(2), q, .. }] if (*q - 0.5).abs() < 1e-9 - ), - "{sql} should lower to Quantile(0.5) over latency, got {measures:?}" - ); - } -} - -#[tokio::test] -async fn median_is_the_same_intent_as_an_explicit_half_percentile() { - // Two spellings of one intent: CSE should be able to merge them. - let m = lower("SELECT median(latency) FROM metrics").await; - let p = lower("SELECT approx_percentile_cont(latency, 0.5) FROM metrics").await; - let (_, m_measures) = find_aggregate(&m).expect("expected an Aggregate"); - let (_, p_measures) = find_aggregate(&p).expect("expected an Aggregate"); - assert_eq!(m_measures, p_measures); -} - -#[tokio::test] -async fn median_threads_the_accuracy_target() { - // The `approx_` prefix does not decide: the AccuracyTarget does. - let qe = lower_sql( - "SELECT approx_median(latency) FROM metrics", - &catalog(), - AccuracyTarget::Epsilon(0.01), - ) - .await - .expect("approx_median should lower"); - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!( - measures.as_slice(), - [AggIntent::Quantile { accuracy: AccuracyTarget::Epsilon(e), .. }] - if (*e - 0.01).abs() < 1e-12 - ), - "median must carry the workload's accuracy target, got {measures:?}" - ); -} - -#[tokio::test] -async fn median_over_an_expression_binds_the_derived_column() { - // Was rejected when filed (#111); supported since #110 materialized the - // expression. What must still hold is the #115 rule: never `col: None`. - let qe = lower("SELECT median(bytes * 8) FROM metrics").await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!(measures.as_slice(), [AggIntent::Quantile { col: Some(_), q, .. }] if (*q - 0.5).abs() < 1e-9), - "expected Quantile(0.5) bound to the derived column, got {measures:?}" - ); -} - -// ── Issue #110: expression GROUP BY (time bucketing) ──────────────────────── - -#[tokio::test] -async fn time_bucketing_group_by_lowers_to_a_derived_key() { - // The canonical time-series shape: `GROUP BY date_trunc(...)`. The bucket - // expression is materialized beneath the aggregate and grouped on. - let qe = - lower("SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics GROUP BY m").await; - let node = find_aggregate_node(&qe).expect("expected an Aggregate"); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = node - else { - unreachable!() - }; - assert!( - matches!(**child, QueryExpr::Project { .. }), - "expected a materializing Project beneath the Aggregate" - ); - let schema = child.output_schema().expect("child schema"); - assert_eq!(reduction, &Reduction::by(vec![0])); - assert!( - schema.fields[0].name.contains("date_trunc"), - "group key should be the projected bucket, got {:?}", - schema.fields[0].name - ); - // The reducer still binds its own column, not the bucket. - assert!(matches!( - measures.as_slice(), - [AggIntent::Sum { col: Some(1) }] - )); -} - -#[tokio::test] -async fn time_bucketing_keeps_the_scan_predicate() { - // The projection is inserted above the scan, so a WHERE clause still folds - // onto the Scan rather than being stranded. - let qe = lower( - "SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics \ - WHERE bytes > 10 GROUP BY m", - ) - .await; - fn scan_has_predicate(qe: &QueryExpr) -> bool { - match qe { - QueryExpr::Scan { predicates, .. } => !predicates.is_empty(), - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => scan_has_predicate(child), - _ => false, - } - } - assert!(scan_has_predicate(&qe), "WHERE should stay on the Scan"); -} - -#[tokio::test] -async fn a_plain_group_by_inserts_no_projection() { - // Queries that lowered before #110 must keep their exact DAG shape — the - // projection appears only when something actually needs materializing. - for q in [ - "SELECT service, SUM(bytes) FROM metrics GROUP BY service", - "SELECT SUM(bytes) FROM metrics", - "SELECT COUNT(*) FROM metrics", - ] { - let qe = lower(q).await; - let QueryExpr::Aggregate { child, .. } = - find_aggregate_node(&qe).expect("expected an Aggregate") - else { - unreachable!() - }; - assert!( - !matches!(**child, QueryExpr::Project { .. }), - "{q} should not gain a projection" - ); - } -} - -#[tokio::test] -async fn a_shared_expression_is_materialized_once() { - let qe = lower("SELECT SUM(bytes * 2), MIN(bytes * 2) FROM metrics").await; - let QueryExpr::Aggregate { - measures, child, .. - } = find_aggregate_node(&qe).expect("expected an Aggregate") - else { - unreachable!() - }; - assert_eq!( - child.output_schema().expect("child schema").fields.len(), - 1, - "the two reducers should share one derived column" - ); - assert_eq!(measures[0].input_cols(), measures[1].input_cols()); -} - -// ── Issue #118: multi-level grouping expands into one Aggregate per level ─── - -/// The branches of the first `Concat` along the single-child spine. -fn merge_branches(qe: &QueryExpr) -> &Vec { - fn find(qe: &QueryExpr) -> Option<&Vec> { - match qe { - QueryExpr::Concat { children, .. } => Some(children), - QueryExpr::Project { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } => find(child), - _ => None, - } - } - find(qe).expect("expected a Concat") -} - -/// `(group keys, column names)` of each merged grouping level. -fn grouping_levels(qe: &QueryExpr) -> Vec<(GroupKeys, Vec)> { - merge_branches(qe) - .iter() - .map(|b| { - let QueryExpr::Project { child, .. } = b else { - panic!("expected a Project per level, got {b:?}"); - }; - let QueryExpr::Aggregate { reduction, .. } = child.as_ref() else { - panic!("expected an Aggregate under the Project, got {child:?}"); - }; - let names = b - .output_schema() - .expect("level schema") - .fields - .iter() - .map(|c| c.name.clone()) - .collect(); - (reduction.expect_reduce().clone(), names) - }) - .collect() -} - -#[tokio::test] -async fn rollup_expands_to_one_aggregate_per_prefix() { - // ROLLUP(a, b) → (a,b), (a), () — three levels, widest first. - let qe = - lower("SELECT service, bytes, SUM(latency) FROM metrics GROUP BY ROLLUP(service, bytes)") - .await; - let levels = grouping_levels(&qe); - let keys: Vec<_> = levels.iter().map(|(by, _)| by.clone()).collect(); - assert_eq!( - keys, - vec![ - GroupKeys::by(vec![1, 3]), - GroupKeys::by(vec![1]), - GroupKeys::none(), - ] - ); -} - -#[tokio::test] -async fn cube_expands_to_the_power_set() { - // CUBE(a, b) → (a,b), (a), (b), () — four levels. - let qe = - lower("SELECT service, bytes, SUM(latency) FROM metrics GROUP BY CUBE(service, bytes)") - .await; - assert_eq!(grouping_levels(&qe).len(), 4); -} - -#[tokio::test] -async fn a_mixed_grouping_set_is_normalized_by_datafusion() { - // `GROUP BY g, ROLLUP(d)` arrives as one GroupingSets, not a plain key - // alongside a grouping set — so there is only one shape to handle. - let qe = - lower("SELECT service, bytes, SUM(latency) FROM metrics GROUP BY service, ROLLUP(bytes)") - .await; - assert_eq!(grouping_levels(&qe).len(), 2); -} - -#[tokio::test] -async fn omitted_grouping_keys_become_typed_nulls() { - // Every level must emit every key — as NULL where the level omits it — or - // `Concat` (which takes the first child's schema) would misdescribe the rest. - // The null is *cast*: a bare Null literal infers as Float64. - let qe = lower("SELECT service, SUM(bytes) FROM metrics GROUP BY ROLLUP(service)").await; - let levels = grouping_levels(&qe); - assert_eq!(levels.len(), 2); - for (_, names) in &levels { - assert_eq!( - names, - &["service".to_string(), "sum(metrics.bytes)".to_string()] - ); - } - - // The `()` level projects `service` as a Utf8 null, not a Float64 one. - let schema = merge_branches(&qe)[1] - .output_schema() - .expect("level schema"); - assert_eq!(schema.fields[0].name, "service"); - assert_eq!( - schema.fields[0].dtype, - DataType::Utf8, - "the omitted key must keep its declared type" - ); -} - -#[tokio::test] -async fn grouping_levels_are_union_compatible() { - let qe = lower( - "SELECT service, bytes, SUM(latency) FROM metrics GROUP BY GROUPING SETS ((service),(bytes),())", - ) - .await; - let shapes: Vec<_> = merge_branches(&qe) - .iter() - .map(|b| { - b.output_schema() - .expect("level schema") - .fields - .iter() - .map(|c| (c.name.clone(), c.dtype.clone())) - .collect::>() - }) - .collect(); - assert!( - shapes.windows(2).all(|w| w[0] == w[1]), - "levels disagree: {shapes:?}" - ); -} - -#[tokio::test] -async fn grouping_function_is_rejected() { - // `__grouping_id` is dropped when the levels are expanded. It is observable - // only through `GROUPING(col)`, so dropping it loses nothing representable — - // this test is what makes that true. - let err = lower_sql( - "SELECT service, SUM(bytes), GROUPING(service) FROM metrics GROUP BY ROLLUP(service)", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("GROUPING() must be rejected while __grouping_id is dropped"); - assert!(format!("{err}").contains("grouping"), "got {err}"); -} - -#[tokio::test] -async fn a_non_column_key_inside_a_grouping_set_is_rejected() { - // The #110 derived-column machinery covers plain `GROUP BY `; inside a - // grouping set the key also has to be reinstatable as a typed null. - let err = lower_sql( - "SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics GROUP BY ROLLUP(m)", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("expression key inside ROLLUP must be rejected"); - assert!( - format!("{err}").contains("non-column key inside a multi-level grouping"), - "got {err}" - ); -} - -#[tokio::test] -async fn multi_level_grouping_composes_with_a_derived_reducer_argument() { - // #110's materializing Project sits beneath every level's Aggregate. - let qe = lower("SELECT service, SUM(bytes * 8) FROM metrics GROUP BY ROLLUP(service)").await; - for b in merge_branches(&qe) { - let QueryExpr::Project { child, .. } = b else { - panic!("expected a Project per level"); - }; - let QueryExpr::Aggregate { - measures, child, .. - } = child.as_ref() - else { - panic!("expected an Aggregate"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::Sum { col: Some(_) }] - )); - assert!( - matches!(**child, QueryExpr::Project { .. }), - "the derived-column projection should sit under each level" - ); - } -} - -#[tokio::test] -async fn an_ambiguous_passthrough_column_is_rejected_only_when_projecting() { - // A `Project` carries one relation qualifier for all its columns, so `a.k` - // and `b.k` cannot both survive it. That only matters once a projection is - // inserted: without a derived column the join keys resolve as before. - let ok = lower_sql( - "SELECT m.service, h.service, SUM(m.bytes) FROM metrics m \ - JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", - &catalog(), - AccuracyTarget::Exact, - ) - .await; - assert!( - ok.is_ok(), - "no derived column ⇒ no projection ⇒ no ambiguity" - ); - - let err = lower_sql( - "SELECT m.service, h.service, SUM(m.bytes * 2) FROM metrics m \ - JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("ambiguous passthrough must be rejected, not silently resolved"); - assert!(format!("{err}").contains("ambiguous column"), "got {err}"); -} - -// ── Issue #111: array_agg is deliberately not an intent (WONTFIX) ─────────── - -#[tokio::test] -async fn array_agg_is_deliberately_rejected() { - // Not a coverage gap. `AggIntent` exists so the planner can bind a sketch or - // a mergeable accumulator per node; `array_agg` pre-aggregates nothing (its - // output is O(input rows)), has no bounded-memory approximate form, and its - // partial state *is* the data. An `AggIntent::ArrayAgg` would force every - // arm of `plan::boundary::realize` — an exhaustive match — to answer - // `PassThrough`. Contrast `median`, which is `Quantile { q: 0.5 }` and does - // feed the sketch path. - // - // This test exists so the rejection reads as a decision rather than a gap. - let err = lower_sql( - "SELECT array_agg(service) FROM metrics", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .expect_err("array_agg must not lower to an intent"); - assert!( - format!("{err}").contains("unsupported aggregate: array_agg"), - "expected a clean UnsupportedAggregate, got {err}" - ); -} - -// ── Issue #225: catalog-driven ClickHouse builtins (countIf, generalizing -// uniqExact from #221) ─────────────────────────────────────────────────── - -async fn lower_clickhouse(sql: &str) -> QueryExpr { - lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) -} - -fn temporal_aggregate(qe: &QueryExpr) -> (&AggIntent, std::time::Duration, &QueryExpr) { - match qe { - QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures, - child, - .. - } => { - let QueryExpr::TimeRange { range, child } = child.as_ref() else { - panic!("temporal Aggregate must directly wrap TimeRange, got {child:?}"); - }; - (&measures[0], *range, child) - } - QueryExpr::Project { child, .. } | QueryExpr::Filter { child, .. } => { - temporal_aggregate(child) - } - other => panic!("expected temporal Aggregate, got {other:?}"), - } -} - -#[tokio::test] -async fn explicit_temporal_aggregates_share_promql_intents_and_timerange() { - for (function, expected) in [ - ("asap_rate", AggIntent::Rate), - ("asap_increase", AggIntent::Increase), - ] { - let sql = format!( - "SELECT service, {function}(latency, ts, 300000) AS v \ - FROM metrics WHERE service = 'api' GROUP BY service" - ); - let qe = lower_clickhouse(&sql).await; - let (intent, range, child) = temporal_aggregate(&qe); - assert_eq!(intent, &expected); - assert_eq!(range, std::time::Duration::from_secs(300)); - assert!(matches!(child, QueryExpr::Project { child, .. } - if matches!(child.as_ref(), QueryExpr::Scan { predicates, .. } if predicates.len() == 1))); - - let QueryExpr::Project { cols, .. } = &qe else { - panic!("SELECT list must remain a Project, got {qe:?}"); - }; - assert!(matches!(cols[0].expr, QueryExpr::Column(2))); - assert_eq!(cols[1].alias.as_deref(), Some("v")); - assert!(matches!(cols[1].expr, QueryExpr::Column(1))); - } -} - -#[tokio::test] -async fn temporal_aggregate_rejects_non_timestamp_and_non_positive_window() { - for sql in [ - "SELECT asap_rate(latency, bytes, 300000) FROM metrics", - "SELECT asap_rate(latency, ts, 0) FROM metrics", - "SELECT asap_rate(latency, ts, bytes) FROM metrics", - ] { - let err = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect_err("invalid temporal arguments must fail closed"); - assert!( - format!("{err}").contains("timestamp argument") - || format!("{err}").contains("window_ms"), - "unexpected error for {sql}: {err}" - ); - } -} - -#[tokio::test] -async fn temporal_aggregate_rejects_mixed_reducers() { - let err = lower_sql_dialect( - "SELECT asap_rate(latency, ts, 300000), sum(bytes) FROM metrics", - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect_err("one child cannot carry temporal and ordinary aggregate semantics"); - assert!(format!("{err}").contains("cannot share an Aggregate node")); -} - -#[tokio::test] -async fn last_fails_closed_until_an_executable_summary_exists() { - let err = lower_sql_dialect( - "SELECT service, asap_last(latency, ts, 300000) FROM metrics GROUP BY service", - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect_err("last must not be advertised without an executable physical summary"); - assert!(format!("{err}").contains("Invalid function 'asap_last'")); -} - -#[tokio::test] -async fn temporal_grouping_requires_the_complete_declared_series_identity() { - let multi_series = SqlCatalog::new().with_table( - "samples", - Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("instance", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![0, 1, 2]], - ), - ); - for sql in [ - "SELECT asap_rate(value, ts, 300000) FROM samples", - "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", - ] { - let err = lower_sql_dialect( - sql, - &multi_series, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect_err("partial identity must not merge counter series"); - assert!(format!("{err}").contains("declared series identity")); - } - - lower_sql_dialect( - "SELECT service, instance, asap_rate(value, ts, 300000) \ - FROM samples GROUP BY service, instance", - &multi_series, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect("the complete declared series identity is safe"); - - let row_id_only = SqlCatalog::new().with_table( - "samples", - Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp), - col("service", DataType::Utf8), - col("value", DataType::Float64), - ], - 0, - vec![vec![1]], - ), - ); - lower_sql_dialect( - "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", - &row_id_only, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect_err("a row key without time does not prove a series identity"); -} - -#[tokio::test] -async fn temporal_grouping_rejects_value_time_and_duplicate_resolved_columns() { - for sql in [ - "SELECT asap_rate(latency, ts, 300000) FROM metrics GROUP BY ts", - "SELECT asap_rate(latency, ts, 300000) FROM metrics GROUP BY latency", - "SELECT m.service, asap_rate(m.latency, m.ts, 300000) \ - FROM metrics m GROUP BY m.service, service", - ] { - let err = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect_err("unsafe or duplicate resolved grouping must fail closed"); - let message = format!("{err}"); - assert!( - message.contains("timestamp or value") - || message.contains("same resolved column more than once"), - "unexpected error for {sql}: {message}" - ); - } -} - -#[tokio::test] -async fn qualified_columns_are_validated_by_resolved_identity() { - let qe = lower_clickhouse( - "SELECT m.service, asap_increase(m.latency, m.ts, 300000) AS v \ - FROM metrics AS m GROUP BY m.service", - ) - .await; - let (intent, range, _) = temporal_aggregate(&qe); - assert_eq!(intent, &AggIntent::Increase); - assert_eq!(range, std::time::Duration::from_secs(300)); -} - -#[tokio::test] -async fn project_filter_and_outer_aggregate_preserve_temporal_child() { - let qe = lower_clickhouse( - "SELECT max(v) FROM (\ - SELECT service, asap_rate(latency, ts, 300000) AS v \ - FROM metrics WHERE bytes > 0 GROUP BY service\ - ) r WHERE v >= 0", - ) - .await; - let QueryExpr::Project { child, .. } = &qe else { - panic!("expected outer SELECT Project, got {qe:?}"); - }; - let QueryExpr::Aggregate { - reduction: Reduction::Reduce(_), - measures, - child, - .. - } = child.as_ref() - else { - panic!("expected outer Aggregate, got {child:?}"); - }; - assert!(matches!(measures.as_slice(), [AggIntent::Max { .. }])); - let QueryExpr::Filter { child, .. } = child.as_ref() else { - panic!("derived-table WHERE must remain above the inner query, got {child:?}"); - }; - let (intent, range, _) = temporal_aggregate(child); - assert_eq!(intent, &AggIntent::Rate); - assert_eq!(range, std::time::Duration::from_secs(300)); -} - -#[tokio::test] -async fn count_if_lowers_to_a_sum_over_a_derived_indicator_column() { - // ClickHouse's `countIf(cond)` has no DataFusion equivalent at all, so it - // goes through the same stub-UDAF + catalog-driven `FunctionRewrite` - // mechanism `uniqExact` (#221) does — rewritten, before `lower_agg_intent` - // ever runs, to `sum(CASE WHEN cond THEN 1 ELSE 0 END)`. A per-measure - // filter (#466) could express it as a filtered `Count` now; that move is - // a follow-up, so the indicator sum is still the shape to expect. - let qe = lower_clickhouse("SELECT countIf(bytes > 100) AS big FROM metrics").await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!(by.is_empty()); - assert!( - matches!(measures.as_slice(), [AggIntent::Sum { col: Some(_) }]), - "expected a Sum bound to the derived indicator column, got {measures:?}" - ); - let (_, materialized) = reducer_input_names(&qe); - assert!( - materialized, - "the indicator expression must be materialized in a Project beneath the Aggregate" - ); -} - -#[tokio::test] -async fn two_count_ifs_with_different_conditions_stay_distinct_reducers() { - // The corpus pattern (`countIf(operation = 'A'), countIf(operation = 'W')` - // in one GROUP BY) needs each call's own condition to survive as its own - // derived column, not collapse onto a shared one. - let qe = lower_clickhouse( - "SELECT service, countIf(bytes > 100) AS big, countIf(bytes <= 100) AS small \ - FROM metrics GROUP BY service", - ) - .await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert_eq!(*by, GroupKeys::by(vec![0])); - assert!( - matches!( - measures.as_slice(), - [ - AggIntent::Sum { col: Some(a) }, - AggIntent::Sum { col: Some(b) } - ] if a != b - ), - "expected two distinct Sum reducers, got {measures:?}" - ); -} - -#[tokio::test] -async fn count_if_composes_with_group_by() { - let qe = lower_clickhouse( - "SELECT service, countIf(bytes > 100) AS big FROM metrics GROUP BY service", - ) - .await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert_eq!(*by, GroupKeys::by(vec![0])); - assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); -} - -// ── Issue #232: argMax/argMin -- AggIntent::Extension, not a first-class -// core variant. A repo-wide search (PromQL front end, other SQL dialects, -// docs) turned up no second deployment model wanting this two-column, -// row-selecting shape, so per `AggIntent::Extension`'s own "core only grows -// for intents ≥2 deployment models actually use" bar, it stays an opaque -// `Extension` rather than a new `ArgMax`/`ArgMin` core variant. Unlike -// `countIf`/`uniqExact`, there is no native DataFusion aggregate shape to -// rewrite to (`RewriteKind::PassThrough`) -- `lower_agg_intent` builds the -// `AggIntent` directly from the ClickHouse name. ───────────────────────── - -#[tokio::test] -async fn arg_max_lowers_to_an_extension_intent() { - // No existing `AggIntent` reducer fits: every one folds one column to a - // value derived from itself, while `argMax(arg, val)` returns a - // *different* column's value, selected by which row maximizes a second. - let qe = lower_clickhouse( - "SELECT service, argMax(service, latency) AS busiest FROM metrics GROUP BY service", - ) - .await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert_eq!( - *by, - GroupKeys::by(vec![1]), - "grouped by `service` (schema index 1)" - ); - assert!( - matches!( - measures.as_slice(), - [AggIntent::Extension { ext_kind, .. }] if ext_kind == "arg_max" - ), - "expected Extension {{ ext_kind: \"arg_max\", .. }}, got {measures:?}" - ); -} - -#[tokio::test] -async fn arg_min_lowers_to_its_own_extension_kind() { - let qe = lower_clickhouse("SELECT argMin(service, latency) FROM metrics").await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!(by.is_empty()); - assert!( - matches!( - measures.as_slice(), - [AggIntent::Extension { ext_kind, .. }] if ext_kind == "arg_min" - ), - "expected Extension {{ ext_kind: \"arg_min\", .. }}, got {measures:?}" - ); -} - -#[tokio::test] -async fn arg_max_payload_preserves_both_column_names() { - // Core never resolves an `Extension`'s payload, so both columns are kept - // as validated bare-column `ColumnRef`s in `payload`, not run through - // positional `ColumnId` binding -- see `lower_arg_selector`'s doc. - let qe = lower_clickhouse("SELECT argMax(service, latency) AS m FROM metrics").await; - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - let AggIntent::Extension { payload, .. } = &measures[0] else { - panic!("expected an Extension intent, got {:?}", measures[0]); - }; - let named = |key: &str| { - payload - .get(key) - .and_then(|c| c.get("Named")) - .and_then(|n| n.as_str()) - .map(str::to_string) - }; - assert_eq!(named("arg_col"), Some("service".to_string())); - assert_eq!(named("val_col"), Some("latency".to_string())); -} - -#[tokio::test] -async fn arg_max_rejects_a_non_column_argument() { - // Same "bare column only" rule as every other reducer (`reducer_col`, - // issue #115) -- an expression argument is rejected, not silently - // dropped or materialized into the wrong column. - let err = lower_sql_dialect( - "SELECT argMax(service, latency * 2) FROM metrics", - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .expect_err("argMax over a non-column expression must be rejected"); - assert!( - format!("{err}").contains("non-column expression"), - "got {err}" - ); -} - -// ── Issue #267: lagInFrame/leadInFrame get distinct WindowFuncKind variants, -// not conflated with ANSI Lag/Lead ────────────────────────────────────────── - -#[tokio::test] -async fn lag_in_frame_lowers_to_its_own_kind_not_lag() { - let qe = lower_clickhouse( - "SELECT service, lagInFrame(bytes) OVER (PARTITION BY service ORDER BY ts) \ - FROM metrics", - ) - .await; - let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); - let QueryExpr::SQLWindowFunc { func, args, .. } = win else { - unreachable!(); - }; - assert_eq!(*func, WindowFuncKind::LagInFrame); - assert_eq!( - args, - &vec![QueryExpr::Column(3)], - "lagInFrame(bytes) → arg col 3" - ); -} - -#[tokio::test] -async fn lead_in_frame_lowers_to_its_own_kind_not_lead() { - let qe = lower_clickhouse( - "SELECT service, leadInFrame(bytes) OVER (PARTITION BY service ORDER BY ts) \ - FROM metrics", - ) - .await; - let win = find_windowfunc(&qe).expect("expected a SQLWindowFunc node"); - let QueryExpr::SQLWindowFunc { func, .. } = win else { - unreachable!(); - }; - assert_eq!(*func, WindowFuncKind::LeadInFrame); -} - -/// Issue #184: `NOW()` in a predicate must lower to the timestamp-typed -/// `CurrentTimestamp` leaf, not the semantically-opaque function catch-all or -/// PromQL's Float64 Unix-seconds `EvalTimestamp`. -#[tokio::test] -async fn now_in_predicate_lowers_to_current_timestamp() { - // WHERE folds onto Scan.predicates (no explicit Filter node). - let qe = lower("SELECT * FROM metrics WHERE ts < NOW()").await; - let QueryExpr::Project { child, .. } = &qe else { - panic!("expected Project at root, got {qe:?}"); - }; - let QueryExpr::Scan { predicates, .. } = child.as_ref() else { - panic!("expected Scan under the projection, got {child:?}"); - }; - assert_eq!(predicates.len(), 1); - assert!( - matches!(predicates[0].0.as_ref(), QueryExpr::Compare { right, .. } - if matches!(right.as_ref(), QueryExpr::CurrentTimestamp)), - "NOW() must lower to CurrentTimestamp, got {:?}", - predicates[0].0 - ); -} - -/// Same for ClickHouse's `now()`, since #184 was raised specifically against -/// the ClickHouse dialect. -#[tokio::test] -async fn clickhouse_now_in_predicate_lowers_to_current_timestamp() { - let qe = lower_clickhouse("SELECT * FROM metrics WHERE ts < now()").await; - let QueryExpr::Project { child, .. } = &qe else { - panic!("expected Project at root, got {qe:?}"); - }; - let QueryExpr::Scan { predicates, .. } = child.as_ref() else { - panic!("expected Scan under the projection, got {child:?}"); - }; - assert_eq!(predicates.len(), 1); - assert!( - matches!(predicates[0].0.as_ref(), QueryExpr::Compare { right, .. } - if matches!(right.as_ref(), QueryExpr::CurrentTimestamp)), - "now() must lower to CurrentTimestamp, got {:?}", - predicates[0].0 - ); -} - -#[tokio::test] -async fn current_timestamp_lowers_to_typed_current_timestamp_leaf() { - let qe = lower("SELECT CURRENT_TIMESTAMP FROM metrics").await; - let QueryExpr::Project { cols, .. } = &qe else { - panic!("expected Project at root, got {qe:?}"); - }; - assert!(matches!(&cols[0].expr, QueryExpr::CurrentTimestamp)); - let schema = cols[0].expr.output_schema().expect("timestamp schema"); - assert_eq!(schema.fields[0].dtype, DataType::Timestamp); -} - -// A `count` over a non-null input is a plain row count; over a nullable -// input it keeps SQL's NULL-skipping as the measure's own filter (#466), and -// only the multi-level grouping path, which cannot carry one, still rejects it. -#[tokio::test] -async fn count_null_semantics_become_a_measure_filter() { - let catalog = SqlCatalog::new().with_table( - "samples", - Schema::new(vec![ - Field::plain("nullable_value", DataType::Float64, true), - Field::plain("value", DataType::Float64, false), - ]), - ); - for sql in [ - "SELECT count(*) FROM samples", - "SELECT count(1) FROM samples", - "SELECT count(value) FROM samples", - "SELECT count(value + 1) FROM samples", - ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::Exact) - .await - .unwrap_or_else(|error| panic!("{sql}: {error}")); - assert!( - aggregate_filters(&qe).is_empty(), - "{sql}: unfiltered row count" - ); - } - for sql in [ - "SELECT count(nullable_value) FROM samples", - "SELECT count(NULL) FROM samples", - "SELECT count(nullable_value + 1) FROM samples", - ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::Exact) - .await - .unwrap_or_else(|error| panic!("{sql}: {error}")); - let [Some(Predicate(cond))] = aggregate_filters(&qe) else { - panic!( - "{sql}: expected one filtered Count, got {:?}", - aggregate_filters(&qe) - ); - }; - assert!( - matches!(cond.as_ref(), QueryExpr::IsNotNull(_)), - "{sql}: {cond:?}" - ); - } - // Only the second measure is filtered. - let qe = lower_sql( - "SELECT count(*), count(nullable_value) FROM samples", - &catalog, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - assert!(matches!(aggregate_filters(&qe), [None, Some(_)])); - let error = lower_sql( - "SELECT count(nullable_value) FROM samples GROUP BY ROLLUP(value)", - &catalog, - AccuracyTarget::Exact, - ) - .await - .unwrap_err(); - assert!( - matches!(error, LoweringError::UnsupportedFeature(_)), - "{error}" - ); -} - -/// A native SQL map grouping key retains its typed key/value schema. -#[tokio::test] -async fn grouped_map_column_preserves_map_type() { - let map = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }; - let catalog = SqlCatalog::new().with_table( - "raw_samples", - Schema::new(vec![ - col("labels", map.clone()), - col("value", DataType::Float64), - ]), - ); - let query = lower_sql_dialect( - "SELECT labels, max(value) AS value FROM raw_samples GROUP BY labels ORDER BY labels", - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - assert_eq!(query.output_schema().unwrap().fields[0].dtype, map); -} - -#[tokio::test] -async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { - let catalog = SqlCatalog::new().with_table( - "numbers", - Schema::new(vec![ - Field::plain("i", DataType::Int64, false), - Field::plain("n", DataType::Int64, true), - Field::plain("f", DataType::Float64, false), - ]), - ); - for (call, native) in [ - ("modulo(i, 3)", "i % 3"), - ("modulo(n, -3)", "n % -3"), - ("modulo(f, 2.5)", "f % 2.5"), - ("modulo(-7, 3)", "-7 % 3"), - ("modulo(i, 0)", "i % 0"), - ("modulo(modulo(i, 5), 2)", "(i % 5) % 2"), - ] { - let function = lower_sql_dialect( - &format!("SELECT {call} AS value FROM numbers"), - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let operator = lower_sql_dialect( - &format!("SELECT {native} AS value FROM numbers"), - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - assert_eq!(function, operator, "{call}"); - assert_eq!( - function.output_schema().unwrap(), - operator.output_schema().unwrap() - ); - } - let nullable = lower_sql_dialect( - "SELECT modulo(n, 3) AS value FROM numbers", - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap() - .output_schema() - .unwrap(); - assert_eq!(nullable.fields[0].dtype, DataType::Int64); - assert!(nullable.fields[0].nullable); -} - -#[tokio::test] -async fn original_o11y_map_queries_lower_with_typed_results() { - let catalog = SqlCatalog::new().with_table( - "raw_samples", - Schema::new(vec![ - Field::plain("metric", DataType::Utf8, false), - Field::plain("ts_ms", DataType::Int64, false), - Field::plain("value", DataType::Float64, false), - Field::plain( - "labels", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }, - false, - ), - ]), - ); - for sql in [ - include_str!("data/o11y_q10.sql"), - include_str!("data/o11y_q27.sql"), - include_str!("data/o11y_q07.sql"), - include_str!("data/o11y_q09.sql"), - include_str!("data/o11y_q12.sql"), - ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); - let schema = query.output_schema().unwrap(); - assert!( - schema - .fields - .iter() - .any(|column| matches!(column.dtype, FieldDataType::Plain(DataType::Map { .. }))), - "{schema:?}" - ); - } -} - -#[tokio::test] -async fn clickhouse_modulo_preserves_projection_names_and_outer_references() { - for (sql, name) in [ - ("SELECT modulo(bytes, 3) FROM metrics", "modulo(bytes, 3)"), - ( - "SELECT modulo(bytes, 3) AS remainder FROM metrics", - "remainder", - ), - ( - "SELECT \"modulo(bytes, 3)\" FROM (SELECT modulo(bytes, 3) FROM metrics) t", - "modulo(bytes, 3)", - ), - ] { - let query = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - assert_eq!(query.output_schema().unwrap().fields[0].name, name); - } -} - -#[tokio::test] -async fn clickhouse_map_access_keeps_generated_names_and_rejects_variant_coercion() { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain( - "labels", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }, - false, - ), - Field::plain("integer", DataType::Int64, false), - Field::plain("floating", DataType::Float64, false), - ]), - ); - let query = lower_sql_dialect( - "SELECT labels['job'] FROM t", - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let output = query.output_schema().unwrap(); - assert_eq!(output.fields[0].name, "arrayElement(labels, 'job')"); - assert_eq!(output.fields[0].dtype, DataType::Utf8); - assert!(!output.fields[0].nullable); - assert!(lower_sql_dialect( - "SELECT map()['a'] FROM t", - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .is_err()); - assert!(lower_sql_dialect( - "SELECT map('a', integer, 'b', floating) FROM t", - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact - ) - .await - .is_err()); -} - -#[tokio::test] -async fn arg_selector_result_schema_tracks_selected_argument() { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("v", DataType::Float64, false), - Field::plain("text", DataType::Utf8, true), - Field::plain("ts", DataType::Int64, true), - ]), - ); - for (sql, dtype, nullable) in [ - ( - "SELECT argMax(v, ts) AS value FROM t", - DataType::Float64, - false, - ), - ( - "SELECT argMin(text, ts) AS value FROM t", - DataType::Utf8, - true, - ), - ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let schema = query.output_schema().unwrap(); - assert_eq!(schema.fields[0].dtype, dtype); - assert_eq!(schema.fields[0].nullable, nullable); - } -} - -#[tokio::test] -async fn clickhouse_list_element_uses_canonical_typed_access() { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain( - "samples", - DataType::List { - element: Box::new(Field::new("item", DataType::Int64, false)), - }, - false, - ), - Field::plain("index", DataType::Int64, true), - ]), - ); - for (sql, nullable) in [ - ("SELECT samples[1] AS selected FROM t", false), - ("SELECT arrayElement(samples, -1) AS selected FROM t", false), - ("SELECT samples[index] AS selected FROM t", true), - ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let output = query.output_schema().unwrap(); - assert_eq!(output.fields[0].dtype, DataType::Int64); - assert_eq!(output.fields[0].nullable, nullable); - let serialized = serde_json::to_string(&query).unwrap(); - assert!(serialized.contains("asap_element_access"), "{serialized}"); - } - for sql in ["SELECT samples[0] FROM t", "SELECT samples['bad'] FROM t"] { - assert!( - lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact - ) - .await - .is_err(), - "{sql}" - ); - } -} - -#[tokio::test] -async fn clickhouse_tuple_element_preserves_declared_field_metadata() { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain( - "sample", - DataType::Struct { - fields: vec![ - Field::new("time", DataType::Int64, false), - Field::new("value", DataType::Float64, true), - ], - }, - false, - ), - Field::plain("index", DataType::Int64, false), - ]), - ); - for (sql, dtype, nullable) in [ - ( - "SELECT tupleElement(sample, 1) AS chosen FROM t", - DataType::Int64, - false, - ), - ( - "SELECT tupleElement(sample, 'value') AS chosen FROM t", - DataType::Float64, - true, - ), - ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let output = query.output_schema().unwrap(); - assert_eq!(output.fields[0].dtype, dtype); - assert_eq!(output.fields[0].nullable, nullable); - assert!(serde_json::to_string(&query) - .unwrap() - .contains("asap_struct_field")); - } - for selector in ["0", "-1", "3", "'missing'", "index"] { - let sql = format!("SELECT tupleElement(sample, {selector}) FROM t"); - assert!( - lower_sql_dialect( - &sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::Exact - ) - .await - .is_err(), - "{sql}" - ); - } -} - -/// Correlation lowers to a nullable numeric result instead of UnsupportedAggregate. -#[tokio::test] -async fn corr_result_is_nullable_float() { - let query = lower("SELECT corr(latency, bytes) AS correlation FROM metrics").await; - let schema = query.output_schema().unwrap(); - assert_eq!(schema.fields[0].name, "correlation"); - assert_eq!(schema.fields[0].dtype, DataType::Float64); - assert!(schema.fields[0].nullable); -} - -// A multi-column DISTINCT counts tuples; one column stays the single-column -// intent, so neither form can be mistaken for the other downstream. -#[tokio::test] -async fn composite_distinct_counts_tuples() { - let cat = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("a", DataType::Int64, false), - Field::plain("b", DataType::Int64, false), - ]), - ); - let composite = lower_sql( - "SELECT COUNT(DISTINCT a, b) FROM t", - &cat, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let QueryExpr::Aggregate { measures, .. } = - find_aggregate_node(&composite).expect("expected an Aggregate") - else { - unreachable!() - }; - assert!( - matches!(measures.as_slice(), [AggIntent::Cardinality { cols, .. }] if cols == &[0, 1]), - "{measures:?}" - ); - - let single = lower_sql( - "SELECT COUNT(DISTINCT a) FROM t", - &cat, - AccuracyTarget::Exact, - ) - .await - .unwrap(); - let QueryExpr::Aggregate { measures, .. } = - find_aggregate_node(&single).expect("expected an Aggregate") - else { - unreachable!() - }; - assert!( - matches!(measures.as_slice(), [AggIntent::Cardinality { cols, .. }] if cols == &[0]), - "{measures:?}" - ); -} - -// An expression argument has no column identity to hash, so it is rejected -// rather than silently reduced over a probe column. -#[tokio::test] -async fn composite_distinct_rejects_expression_arguments() { - let cat = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("a", DataType::Int64, false), - Field::plain("b", DataType::Int64, false), - ]), - ); - let error = lower_sql( - "SELECT COUNT(DISTINCT a, b + 1) FROM t", - &cat, - AccuracyTarget::Exact, - ) - .await - .unwrap_err(); - assert!( - matches!(&error, LoweringError::UnsupportedAggregate(reason) - if reason.contains("non-column expression")), - "{error}" - ); -} - -// DISTINCT inputs survive projections introduced by sibling aggregates. -#[tokio::test] -async fn distinct_with_derived_sibling() { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("a", DataType::Int64, false), - Field::plain("b", DataType::Int64, false), - ]), - ); - for sql in [ - "SELECT count(DISTINCT a), sum(b + 1) FROM t", - "SELECT count(DISTINCT a, b), sum(b + 1) FROM t", - "SELECT count(DISTINCT a, b), corr(a,b) FROM t", - ] { - let result = lower_sql(sql, &catalog, AccuracyTarget::Exact).await; - assert!(result.is_ok(), "{sql}: {result:?}"); - } -} - -// ── Issue #466: per-measure FILTER predicates ───────────────────────────────── - -/// The first `Aggregate`'s `filters`, positional against its child. -fn aggregate_filters(qe: &QueryExpr) -> &[Option] { - let Some(QueryExpr::Aggregate { filters, .. }) = find_aggregate_node(qe) else { - panic!("expected an Aggregate, got {qe:?}"); - }; - filters -} - -// The motivating query: one scan, one grouping, one conditional count next to -// a plain sum — a single `Aggregate` whose Count carries the condition, with no -// `Join` and no derived column for the `CASE`. -#[tokio::test] -async fn conditional_count_lowers_to_a_filtered_measure() { - let qe = lower( - "SELECT service, count(CASE WHEN latency > 1.0 THEN 1 END), sum(bytes) \ - FROM metrics GROUP BY service", - ) - .await; - assert!(find_join(&qe).is_none(), "no join: {qe:?}"); - let (by, measures) = find_aggregate(&qe).unwrap(); - assert_eq!(by.keys(), &[1]); - assert!( - matches!( - measures.as_slice(), - [AggIntent::Count { .. }, AggIntent::Sum { col: Some(3) }] - ), - "{measures:?}" - ); - let [Some(Predicate(cond)), None] = aggregate_filters(&qe) else { - panic!("expected [Some, None], got {:?}", aggregate_filters(&qe)); - }; - assert!( - matches!(cond.as_ref(), QueryExpr::Compare { left, op: CompareOpKind::Gt, .. } - if matches!(left.as_ref(), QueryExpr::Column(2))), - "latency > 1.0 against the scan, got {cond:?}" - ); - let Some(QueryExpr::Aggregate { child, .. }) = find_aggregate_node(&qe) else { - unreachable!() - }; - assert!( - matches!(child.as_ref(), QueryExpr::Scan { .. }), - "{child:?}" - ); -} - -// `FILTER (WHERE …)` parses under the DataFusion dialect and lands on exactly -// the measure it annotates. -#[tokio::test] -async fn filter_clause_lowers_to_a_measure_filter() { - let qe = lower("SELECT sum(bytes) FILTER (WHERE service = 'a'), count(*) FROM metrics").await; - let [Some(Predicate(cond)), None] = aggregate_filters(&qe) else { - panic!("expected [Some, None], got {:?}", aggregate_filters(&qe)); - }; - assert!( - matches!(cond.as_ref(), QueryExpr::Compare { left, op: CompareOpKind::Eq, right } - if matches!(left.as_ref(), QueryExpr::Column(1)) - && matches!(right.as_ref(), QueryExpr::Literal(ScalarValue::Utf8(s)) if s == "a")), - "{cond:?}" - ); -} - -// SQL `count(expr)` skips NULLs; canonical `Count` counts rows and never sees -// `expr`, so a nullable argument becomes the measure filter `expr IS NOT NULL` -// instead of being rejected (the pre-#466 behavior) or silently over-counted. -#[tokio::test] -async fn count_of_a_nullable_expression_filters_nulls() { - let qe = lower("SELECT count(nullif(bytes, 0)) FROM metrics").await; - let [Some(Predicate(cond))] = aggregate_filters(&qe) else { - panic!("expected [Some], got {:?}", aggregate_filters(&qe)); - }; - assert!(matches!(cond.as_ref(), QueryExpr::IsNotNull(_)), "{cond:?}"); - assert!( - matches!( - find_aggregate(&qe).unwrap().1.as_slice(), - [AggIntent::Count { .. }] - ), - "still a row count" - ); -} - -// The columns a measure filter reads must survive the derived-column -// `Project` a reducer expression inserts beneath the aggregate. -#[tokio::test] -async fn measure_filter_columns_survive_a_derived_column_projection() { - let qe = lower("SELECT sum(bytes * 2) FILTER (WHERE latency > 1.0) FROM metrics").await; - let Some(QueryExpr::Aggregate { child, .. }) = find_aggregate_node(&qe) else { - unreachable!() - }; - assert!( - matches!(child.as_ref(), QueryExpr::Project { .. }), - "{child:?}" - ); - let [Some(Predicate(cond))] = aggregate_filters(&qe) else { - panic!("expected [Some], got {:?}", aggregate_filters(&qe)); - }; - let QueryExpr::Compare { left, .. } = cond.as_ref() else { - panic!("{cond:?}"); - }; - let QueryExpr::Column(id) = left.as_ref() else { - panic!("{left:?}"); - }; - assert_eq!(child.output_schema().unwrap().fields[*id].name, "latency"); -} - -// `GROUP BY ROLLUP` fans one measure list out into one `Aggregate` per level; -// a filtered measure there is rejected rather than silently unfiltered. -#[tokio::test] -async fn measure_filter_inside_a_rollup_is_rejected() { - let err = lower_sql( - "SELECT service, count(*) FILTER (WHERE latency > 1.0) FROM metrics GROUP BY ROLLUP(service)", - &catalog(), - AccuracyTarget::Exact, - ) - .await - .unwrap_err(); - assert!(matches!(err, LoweringError::UnsupportedFeature(_)), "{err}"); -} From 1a431dba860d612d1848482eaa9e9deaf8ff1f21 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:04:17 +0000 Subject: [PATCH 2/3] refactor: move the unified frontends and physical planner to the crate roots Moves plus import updates: `asap_frontend_{sql,promql,metricsql}::unified` become the crate roots, `unified_physical_planner` / `unified_sources` / `expressions::unified_planner` become `physical_planner` / `sources` / `expressions::planner`, and the `unified_*` tests take the names of the tests they replace. The moved files also take the DAG wording ("tree" -> "DAG", "readout" -> "evaluation") the rest of the stack uses, and the MetricsQL frontend's legacy lowering test is ported to the unified API with the ingestion-interval parameter. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- .../src/query_physical_lowering.rs | 2 +- crates/asap-aware-mapping/src/rollup.rs | 2 +- crates/asap-aware-mapping/src/test_support.rs | 2 +- crates/asap-physical-operators/README.md | 17 +++-- crates/asap-physical-operators/src/dag/mod.rs | 4 +- .../src/expressions/mod.rs | 16 ++--- .../{unified_planner.rs => planner.rs} | 0 crates/asap-physical-operators/src/lib.rs | 5 +- .../src/operators/joins/mod.rs | 6 +- .../src/operators/mod.rs | 2 +- .../candidates.rs | 6 +- .../compiled.rs | 0 .../logical.rs | 0 .../mod.rs | 6 +- .../precompute.rs | 0 .../promql_rows.rs | 0 .../promql_values.rs | 0 .../row_values.rs | 0 .../{unified_sources => sources}/memory.rs | 0 .../src/{unified_sources => sources}/mod.rs | 2 +- .../tests/{unified_common => common}/mod.rs | 0 .../tests/current_series_heap.rs | 15 ++-- .../tests/deployment_computation.rs | 5 +- .../{unified_node_ids.rs => node_ids.rs} | 3 +- .../tests/physical_dag.rs | 16 ++--- .../tests/physical_plan_recovery.rs | 4 +- .../tests/physical_semantics.rs | 10 +-- .../tests/plan_properties.rs | 2 +- .../tests/planspace_series_identity_heap.rs | 5 +- .../{unified_precompute.rs => precompute.rs} | 5 +- .../tests/precompute_candidates.rs | 28 ++++---- .../tests/precompute_population.rs | 2 +- .../tests/promql_binary.rs | 6 +- .../tests/promql_values.rs | 4 +- .../asap-physical-operators/tests/raw_scan.rs | 4 +- .../tests/summary_projection.rs | 2 +- .../tests/weighted_topk_binding.rs | 69 +++++++++---------- .../devtools/examples/canonical_examples.rs | 2 +- crates/devtools/src/bin/analyze_corpora.rs | 2 +- crates/devtools/src/bin/sketch_coverage.rs | 2 +- crates/devtools/src/bin/variant_coverage.rs | 2 +- crates/devtools/src/lib.rs | 6 +- .../src/{unified/mod.rs => lib.rs} | 0 crates/frontend-metricsql/tests/lowering.rs | 56 +++++++++------ ...fied_lowering.rs => lowering_semantics.rs} | 6 +- crates/frontend-promql/Cargo.toml | 5 +- .../src/{unified => }/error.rs | 2 +- .../src/{unified => }/histogram.rs | 0 .../src/{unified/mod.rs => lib.rs} | 0 .../src/{unified => }/promql.rs | 34 ++++----- .../frontend-promql/tests/count_planning.rs | 1 - ...gram_metadata.rs => histogram_metadata.rs} | 5 +- .../tests/maintained_population_horizon.rs | 1 - .../awesome_prometheus_alerts.rs | 4 +- .../observability/metrics_observability.rs | 4 +- .../tests/observability/promql_corpus.rs | 6 +- .../tests/promql_binding_regressions.rs | 1 - ...l_conformance.rs => promql_conformance.rs} | 19 +++-- .../tests/promql_equivalence.rs | 1 - ..._promql_lowering.rs => promql_lowering.rs} | 39 +++++------ ...fied_scalar_design.rs => scalar_design.rs} | 3 +- .../tests/{unified_support.rs => support.rs} | 4 +- .../tests/univmon_candidates.rs | 1 - .../frontend-sql/src/{unified => }/error.rs | 0 .../src/{unified/mod.rs => lib.rs} | 2 +- .../src/{unified => }/sql/clickhouse_ast.rs | 0 .../{unified => }/sql/collection_planning.rs | 2 +- .../src/{unified => }/sql/expr.rs | 4 +- .../frontend-sql/src/{unified => }/sql/mod.rs | 6 +- .../src/{unified => }/sql/types.rs | 4 +- .../tests/bgp_analytics/bgp_analytics.rs | 2 +- .../bgp_jan2024_workload.rs | 2 +- .../synthetic_packet_trace.rs | 2 +- .../tests/maintained_population.rs | 2 +- crates/frontend-sql/tests/netflow/netflow.rs | 2 +- crates/frontend-sql/tests/pearson_corr.rs | 2 +- ...nified_sql_lowering.rs => sql_lowering.rs} | 22 +++--- crates/frontend-sql/tests/temporal_types.rs | 2 +- crates/integration-tests/src/lib.rs | 14 ++-- .../tests/frontend_timestamps.rs | 2 +- .../tests/kll_pane_execution.rs | 2 +- .../tests/operator_design_examples.rs | 8 +-- .../tests/operator_sharing.rs | 2 +- .../tests/physical_common/mod.rs | 2 +- .../tests/precompute_raw_samples.rs | 4 +- .../tests/sql_to_physical.rs | 6 +- .../tests/sql_to_post_asap.rs | 2 +- .../summary_maintenance_lifecycle_e2e.rs | 61 ++++++++-------- crates/planner/src/lib.rs | 6 +- crates/planner/tests/e2e_plan.rs | 2 +- crates/planner/tests/summary_sharing.rs | 4 +- crates/sql-function-catalog/src/lib.rs | 2 +- 92 files changed, 295 insertions(+), 332 deletions(-) rename crates/asap-physical-operators/src/expressions/{unified_planner.rs => planner.rs} (100%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/candidates.rs (98%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/compiled.rs (100%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/logical.rs (100%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/mod.rs (99%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/precompute.rs (100%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/promql_rows.rs (100%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/promql_values.rs (100%) rename crates/asap-physical-operators/src/{unified_physical_planner => physical_planner}/row_values.rs (100%) rename crates/asap-physical-operators/src/{unified_sources => sources}/memory.rs (100%) rename crates/asap-physical-operators/src/{unified_sources => sources}/mod.rs (99%) rename crates/asap-physical-operators/tests/{unified_common => common}/mod.rs (100%) rename crates/asap-physical-operators/tests/{unified_node_ids.rs => node_ids.rs} (94%) rename crates/asap-physical-operators/tests/{unified_precompute.rs => precompute.rs} (92%) rename crates/frontend-metricsql/src/{unified/mod.rs => lib.rs} (100%) rename crates/frontend-metricsql/tests/{unified_lowering.rs => lowering_semantics.rs} (94%) rename crates/frontend-promql/src/{unified => }/error.rs (97%) rename crates/frontend-promql/src/{unified => }/histogram.rs (100%) rename crates/frontend-promql/src/{unified/mod.rs => lib.rs} (100%) rename crates/frontend-promql/src/{unified => }/promql.rs (99%) rename crates/frontend-promql/tests/{unified_histogram_metadata.rs => histogram_metadata.rs} (96%) rename crates/frontend-promql/tests/{unified_promql_conformance.rs => promql_conformance.rs} (99%) rename crates/frontend-promql/tests/{unified_promql_lowering.rs => promql_lowering.rs} (98%) rename crates/frontend-promql/tests/{unified_scalar_design.rs => scalar_design.rs} (98%) rename crates/frontend-promql/tests/{unified_support.rs => support.rs} (97%) rename crates/frontend-sql/src/{unified => }/error.rs (100%) rename crates/frontend-sql/src/{unified/mod.rs => lib.rs} (98%) rename crates/frontend-sql/src/{unified => }/sql/clickhouse_ast.rs (100%) rename crates/frontend-sql/src/{unified => }/sql/collection_planning.rs (99%) rename crates/frontend-sql/src/{unified => }/sql/expr.rs (99%) rename crates/frontend-sql/src/{unified => }/sql/mod.rs (99%) rename crates/frontend-sql/src/{unified => }/sql/types.rs (99%) rename crates/frontend-sql/tests/{unified_sql_lowering.rs => sql_lowering.rs} (99%) diff --git a/crates/asap-aware-mapping/src/query_physical_lowering.rs b/crates/asap-aware-mapping/src/query_physical_lowering.rs index 31ccb479a..150104a04 100644 --- a/crates/asap-aware-mapping/src/query_physical_lowering.rs +++ b/crates/asap-aware-mapping/src/query_physical_lowering.rs @@ -2828,7 +2828,7 @@ mod tests { ..Default::default() }), }; - let root = asap_frontend_promql::unified::lower_promql_workload(&workload, 0) + let root = asap_frontend_promql::lower_promql_workload(&workload, 0) .unwrap() .remove(0); // Read one node through every reference (as a planner that shares diff --git a/crates/asap-aware-mapping/src/rollup.rs b/crates/asap-aware-mapping/src/rollup.rs index b39dab6ca..b1e1201b0 100644 --- a/crates/asap-aware-mapping/src/rollup.rs +++ b/crates/asap-aware-mapping/src/rollup.rs @@ -51,7 +51,7 @@ //! here, so it is deliberately left unhandled (see [`rollup_combinator`]). //! In practice this never matters: `Rate`/`Increase` are constructed with //! `Reduction::PerEntity` (per-series, no `by` at all — see -//! `AggIntent::is_per_series` and `asap_frontend_promql::unified::promql::reduction_for`), +//! `AggIntent::is_per_series` and `asap_frontend_promql::promql::reduction_for`), //! never `Reduction::Reduce`, so they never carry a `by` set for this //! strategy to compare in the first place; `PerEntity` nodes never match //! ([`bindable_grouped_aggregate`] returns `None` for them). `Increase`'s diff --git a/crates/asap-aware-mapping/src/test_support.rs b/crates/asap-aware-mapping/src/test_support.rs index d79206dd7..4147d05eb 100644 --- a/crates/asap-aware-mapping/src/test_support.rs +++ b/crates/asap-aware-mapping/src/test_support.rs @@ -35,7 +35,7 @@ pub(crate) fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc(()) ``` -`physical_planner::compile` accepts a logical Post-ASAP DAG (`PostAsapDAG`) and typed input contracts. +`physical_planner::compile` accepts a logical Post-ASAP DAG (`PhysicalASAPDAG`) and typed input contracts. The resulting candidate is instantiated with deployment readers after selection. It rejects unsupported operations and schema mismatches before starting a source. Implement `PhysicalOperator` for a deployment source, including asynchronous I/O; computation operators remain in the library. The public `planner` export identifies the exact Planner types used by the crate. The physical compiler currently supports a subset of those types and -operations; it does not interpret an unknown node as external fallback. -`unified_physical_planner::compile` does the same for a `PhysicalASAPDAG`: a +operations; it does not interpret an unknown node as external fallback: a node without a native lowering fails compilation, and the deployment runtime must execute it. @@ -62,7 +61,7 @@ Plain values preserve Planner scalar/collection types and nullability. Numeric arithmetic uses matching Int64 or Float64 inputs; integer overflow is an error. Boolean predicates use three-valued logic. Native summary states currently cover exact Sum/Count/Min/Max/Rate/Increase, KLL, DDSketch, HLL and Float64 weighted CMS and CountSketch with candidate heaps. Binding checks family, -parameters and readout compatibility; source batches also validate state payloads. +parameters and evaluation compatibility; source batches also validate state payloads. Existing accumulator algorithms are reused as kernels behind these operators. This crate is owned by ASAPPlanner. Its `planner-types` dependency is the local @@ -81,9 +80,9 @@ See [the design](../../docs/design_docs/physical-planning-and-deployment.md). - `operators`: projection, filter, joins, aggregate/window, sort, limit and summary implementations. - `sources`: raw-source interface, Scan and the memory connector. - `physical_planner`: native operator lowering, typed input contracts and checked instantiation. -- `summary_kernels`: in-memory summary state over `asap_sketchlib` and exact Planner state: merge, typed readout and update adapters. -- `readout`: readouts over merged exact summary states. -- `capability`: explicit kernel, native-batch and typed readout validation. +- `summary_kernels`: in-memory summary state over `asap_sketchlib` and exact Planner state: merge, typed evaluation and update adapters. +- `evaluation`: evaluations over merged exact summary states. +- `capability`: explicit kernel, native-batch and typed evaluation validation. The `dag`, `factory`, `traits` and `arithmetic` paths are re-exports. They contain no alternative execution implementations. @@ -107,7 +106,7 @@ There is no spill or partitioned parallel execution in this implementation. ## Physical compilation and deployment inputs -`physical_planner::compile` accepts a Planner `PostAsapDAG`, typed +`physical_planner::compile` accepts a Planner `PhysicalASAPDAG`, typed `InputContract`s and output roots. It returns a reusable `CompiledPhysicalDAG` containing selected native operators and no live readers. Compilation validates schemas, input ordering, sharing and boundedness before deployment source access. diff --git a/crates/asap-physical-operators/src/dag/mod.rs b/crates/asap-physical-operators/src/dag/mod.rs index 1cf3ad1f4..74835cd98 100644 --- a/crates/asap-physical-operators/src/dag/mod.rs +++ b/crates/asap-physical-operators/src/dag/mod.rs @@ -5,6 +5,4 @@ pub use crate::runtime::{ Input, Limits, OutputStream, Reservation, RunContext, Scope, SharedValue, }; pub use crate::Error; -pub use crate::{ - expressions, operators, unified_physical_planner as planner, unified_sources as scan, values, -}; +pub use crate::{expressions, operators, physical_planner as planner, sources as scan, values}; diff --git a/crates/asap-physical-operators/src/expressions/mod.rs b/crates/asap-physical-operators/src/expressions/mod.rs index 738b3eab2..aad035516 100644 --- a/crates/asap-physical-operators/src/expressions/mod.rs +++ b/crates/asap-physical-operators/src/expressions/mod.rs @@ -6,7 +6,8 @@ use crate::{ use planner_types::pre_asap::{ArithmeticOpKind, DataType}; pub mod arithmetic; pub mod binary; -pub mod unified_planner; +mod planner; +pub use planner::CompiledExpression; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] pub enum Expression { Binary { @@ -14,7 +15,7 @@ pub enum Expression { left: Box, right: Box, }, - UnifiedPlanner(Box), + Planner(Box), Column(usize), ExactFloat64(usize), FiniteFloat64(Box), @@ -53,8 +54,8 @@ pub enum Expression { IsNull(Box), } impl Expression { - pub fn unified_planner(expression: unified_planner::CompiledExpression) -> Self { - Self::UnifiedPlanner(Box::new(expression)) + pub fn planner(expression: crate::expressions::CompiledExpression) -> Self { + Self::Planner(Box::new(expression)) } pub(crate) fn dtype(&self, input: &SchemaRef) -> Result<(DataType, bool), Error> { use Expression::*; @@ -100,7 +101,7 @@ impl Expression { }; Ok((dtype, n || m)) } - UnifiedPlanner(expression) => { + Planner(expression) => { expression.validate_input(input)?; Ok(expression.dtype()) } @@ -286,7 +287,7 @@ impl Expression { arithmetic::evaluate_binary(operator, a, b)? } } - UnifiedPlanner(expression) => expression.evaluate(row)?, + Planner(expression) => expression.evaluate(row)?, Label { column, name } => { let Value::Map(entries) = &row[*column] else { return Err(invalid("label read requires a map")); @@ -319,8 +320,7 @@ impl Expression { } } Value::Utf8( - crate::unified_physical_planner::promql_rows::encode_series_identity(&labels)? - .into(), + crate::physical_planner::promql_rows::encode_series_identity(&labels)?.into(), ) } Column(i) => row[*i].clone(), diff --git a/crates/asap-physical-operators/src/expressions/unified_planner.rs b/crates/asap-physical-operators/src/expressions/planner.rs similarity index 100% rename from crates/asap-physical-operators/src/expressions/unified_planner.rs rename to crates/asap-physical-operators/src/expressions/planner.rs diff --git a/crates/asap-physical-operators/src/lib.rs b/crates/asap-physical-operators/src/lib.rs index a68af5a50..ea6b73eaf 100644 --- a/crates/asap-physical-operators/src/lib.rs +++ b/crates/asap-physical-operators/src/lib.rs @@ -26,9 +26,8 @@ mod error; pub use error::Error; pub mod expressions; pub mod operators; +pub mod physical_planner; pub mod plan; pub mod runtime; +pub mod sources; pub mod values; - -pub mod unified_physical_planner; -pub mod unified_sources; diff --git a/crates/asap-physical-operators/src/operators/joins/mod.rs b/crates/asap-physical-operators/src/operators/joins/mod.rs index e494c3cdc..652fbcdcc 100644 --- a/crates/asap-physical-operators/src/operators/joins/mod.rs +++ b/crates/asap-physical-operators/src/operators/joins/mod.rs @@ -60,10 +60,8 @@ impl Operator { use planner_types::pre_asap::JoinKind; let mut joined = left.fields.clone(); joined.extend(right.fields.clone()); - let predicate = crate::expressions::unified_planner::CompiledExpression::compile( - &predicate.0, - &schema(joined.clone()), - )?; + let predicate = + crate::expressions::CompiledExpression::compile(&predicate.0, &schema(joined.clone()))?; if predicate.dtype().0 != DataType::Bool { return Err(invalid("join predicate must be boolean")); } diff --git a/crates/asap-physical-operators/src/operators/mod.rs b/crates/asap-physical-operators/src/operators/mod.rs index 4c2bb0723..517019118 100644 --- a/crates/asap-physical-operators/src/operators/mod.rs +++ b/crates/asap-physical-operators/src/operators/mod.rs @@ -130,7 +130,7 @@ enum Kind { }, Join { kind: planner_types::pre_asap::JoinKind, - predicate: Box, + predicate: Box, }, SummaryBuild { family: SummaryFamilyType, diff --git a/crates/asap-physical-operators/src/unified_physical_planner/candidates.rs b/crates/asap-physical-operators/src/physical_planner/candidates.rs similarity index 98% rename from crates/asap-physical-operators/src/unified_physical_planner/candidates.rs rename to crates/asap-physical-operators/src/physical_planner/candidates.rs index 3a72061ea..90f4c4132 100644 --- a/crates/asap-physical-operators/src/unified_physical_planner/candidates.rs +++ b/crates/asap-physical-operators/src/physical_planner/candidates.rs @@ -368,7 +368,7 @@ mod tests { ..Default::default() }), }; - let root = asap_frontend_promql::unified::lower_promql_workload(&workload, 0) + let root = asap_frontend_promql::lower_promql_workload(&workload, 0) .unwrap() .remove(0); let root = promql_rows::with_series_identity(&root).unwrap(); @@ -401,7 +401,7 @@ mod tests { #[test] fn candidates_for_all_frontiers_share_one_lowering() { let (dag, inputs, root) = grouped_rate(); - let lowered = || crate::unified_physical_planner::LOWERED_NODES.with(|count| count.get()); + let lowered = || crate::physical_planner::LOWERED_NODES.with(|count| count.get()); let before = lowered(); let compiled = compile(&dag, inputs, &[root]).unwrap(); let once = lowered() - before; @@ -455,7 +455,7 @@ mod tests { let (retained, _, root) = grouped_rate(); let ephemeral = with_timing(&retained, |_| QueryTime); let inputs = raw_input(&retained); - let lowered = || crate::unified_physical_planner::LOWERED_NODES.with(|count| count.get()); + let lowered = || crate::physical_planner::LOWERED_NODES.with(|count| count.get()); let before = lowered(); let compiled = compile(&ephemeral, inputs.clone(), &[root]).unwrap(); let once = lowered() - before; diff --git a/crates/asap-physical-operators/src/unified_physical_planner/compiled.rs b/crates/asap-physical-operators/src/physical_planner/compiled.rs similarity index 100% rename from crates/asap-physical-operators/src/unified_physical_planner/compiled.rs rename to crates/asap-physical-operators/src/physical_planner/compiled.rs diff --git a/crates/asap-physical-operators/src/unified_physical_planner/logical.rs b/crates/asap-physical-operators/src/physical_planner/logical.rs similarity index 100% rename from crates/asap-physical-operators/src/unified_physical_planner/logical.rs rename to crates/asap-physical-operators/src/physical_planner/logical.rs diff --git a/crates/asap-physical-operators/src/unified_physical_planner/mod.rs b/crates/asap-physical-operators/src/physical_planner/mod.rs similarity index 99% rename from crates/asap-physical-operators/src/unified_physical_planner/mod.rs rename to crates/asap-physical-operators/src/physical_planner/mod.rs index 4763eb24a..7ba0074d9 100644 --- a/crates/asap-physical-operators/src/unified_physical_planner/mod.rs +++ b/crates/asap-physical-operators/src/physical_planner/mod.rs @@ -80,7 +80,7 @@ pub fn bind_with_data_sources<'a>( dag: &PhysicalASAPDAG, mut sources: BTreeMap>, roots: &[NodeId], - data_sources: &crate::unified_sources::DataSources, + data_sources: &crate::sources::DataSources, ) -> Result, Error> { let restored = logical::restore(dag)?; // Only resolve scans reachable below the selected input boundaries. @@ -906,8 +906,8 @@ fn expression( input: &SchemaRef, ) -> Result { let expr = local_scalar(expr)?; - Ok(Expression::unified_planner( - crate::expressions::unified_planner::CompiledExpression::compile(&expr, input)?, + Ok(Expression::planner( + crate::expressions::CompiledExpression::compile(&expr, input)?, )) } diff --git a/crates/asap-physical-operators/src/unified_physical_planner/precompute.rs b/crates/asap-physical-operators/src/physical_planner/precompute.rs similarity index 100% rename from crates/asap-physical-operators/src/unified_physical_planner/precompute.rs rename to crates/asap-physical-operators/src/physical_planner/precompute.rs diff --git a/crates/asap-physical-operators/src/unified_physical_planner/promql_rows.rs b/crates/asap-physical-operators/src/physical_planner/promql_rows.rs similarity index 100% rename from crates/asap-physical-operators/src/unified_physical_planner/promql_rows.rs rename to crates/asap-physical-operators/src/physical_planner/promql_rows.rs diff --git a/crates/asap-physical-operators/src/unified_physical_planner/promql_values.rs b/crates/asap-physical-operators/src/physical_planner/promql_values.rs similarity index 100% rename from crates/asap-physical-operators/src/unified_physical_planner/promql_values.rs rename to crates/asap-physical-operators/src/physical_planner/promql_values.rs diff --git a/crates/asap-physical-operators/src/unified_physical_planner/row_values.rs b/crates/asap-physical-operators/src/physical_planner/row_values.rs similarity index 100% rename from crates/asap-physical-operators/src/unified_physical_planner/row_values.rs rename to crates/asap-physical-operators/src/physical_planner/row_values.rs diff --git a/crates/asap-physical-operators/src/unified_sources/memory.rs b/crates/asap-physical-operators/src/sources/memory.rs similarity index 100% rename from crates/asap-physical-operators/src/unified_sources/memory.rs rename to crates/asap-physical-operators/src/sources/memory.rs diff --git a/crates/asap-physical-operators/src/unified_sources/mod.rs b/crates/asap-physical-operators/src/sources/mod.rs similarity index 99% rename from crates/asap-physical-operators/src/unified_sources/mod.rs rename to crates/asap-physical-operators/src/sources/mod.rs index 9dbe62c15..9713c2a6f 100644 --- a/crates/asap-physical-operators/src/unified_sources/mod.rs +++ b/crates/asap-physical-operators/src/sources/mod.rs @@ -1,6 +1,6 @@ //! Raw data access. Connectors provide rows; Scan owns Planner predicate semantics. use crate::{ - expressions::unified_planner::CompiledExpression, + expressions::CompiledExpression, plan::PhysicalOperator, runtime::{Input, OutputStream, RunContext}, values::{Batch, SchemaRef, Value}, diff --git a/crates/asap-physical-operators/tests/unified_common/mod.rs b/crates/asap-physical-operators/tests/common/mod.rs similarity index 100% rename from crates/asap-physical-operators/tests/unified_common/mod.rs rename to crates/asap-physical-operators/tests/common/mod.rs diff --git a/crates/asap-physical-operators/tests/current_series_heap.rs b/crates/asap-physical-operators/tests/current_series_heap.rs index e879d7aeb..1f5d5ab2d 100644 --- a/crates/asap-physical-operators/tests/current_series_heap.rs +++ b/crates/asap-physical-operators/tests/current_series_heap.rs @@ -1,13 +1,12 @@ //! Spatial heap weights come from a fresh instant vector, never sample history. -#[path = "unified_common/mod.rs"] mod common; use asap_physical_operators::{ operators::Operator, - runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{ + physical_planner::{ promql_rows::{decode_series_identity, series_row, SERIES_IDENTITY_COLUMN}, CompiledPhysicalDAG, InputContract, Source, }, + runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; use common::compile_physical_asap_dag; @@ -273,7 +272,7 @@ fn current_series_observes_resource_limits() { #[test] fn identity_encoding_is_lossless_and_rejects_noncanonical_inputs() { - use asap_physical_operators::unified_physical_planner::promql_rows::encode_series_identity; + use asap_physical_operators::physical_planner::promql_rows::encode_series_identity; let labels = BTreeMap::from([ ("a".into(), "quote\"slash\\".into()), ("other".into(), "".into()), @@ -296,9 +295,7 @@ fn identity_encoding_is_lossless_and_rejects_noncanonical_inputs() { // test does not manually assemble the computation or its dependency edges. #[test] fn planner_current_series_candidate_compiles_with_dynamic_identity() { - use asap_physical_operators::unified_physical_planner::{ - compile, promql_rows::with_series_identity, - }; + use asap_physical_operators::physical_planner::{compile, promql_rows::with_series_identity}; use planner_types::{types::AccuracyTarget, workload::*}; use std::rc::Rc; let workload = PlanningWorkload { @@ -325,7 +322,7 @@ fn planner_current_series_candidate_compiles_with_dynamic_identity() { ..Default::default() }), }; - let original = asap_frontend_promql::unified::lower_promql_workload(&workload, 0) + let original = asap_frontend_promql::lower_promql_workload(&workload, 0) .unwrap() .remove(0); let open_root = Rc::new(original.clone()); @@ -336,7 +333,7 @@ fn planner_current_series_candidate_compiles_with_dynamic_identity() { .candidate(&open_root) .unwrap(); let snapshot_program = - asap_physical_operators::unified_physical_planner::promql_rows::compile_current_series_evaluation( + asap_physical_operators::physical_planner::promql_rows::compile_current_series_evaluation( &open_selected, ) .unwrap(); diff --git a/crates/asap-physical-operators/tests/deployment_computation.rs b/crates/asap-physical-operators/tests/deployment_computation.rs index fbb23f049..4e60ae5c7 100644 --- a/crates/asap-physical-operators/tests/deployment_computation.rs +++ b/crates/asap-physical-operators/tests/deployment_computation.rs @@ -1,11 +1,10 @@ //! Planner-selected PromQL computation compiles from the timed DAG alone; //! the deployment supplies only raw rows at the ingestion frontier. -#[path = "unified_common/mod.rs"] mod common; use asap_physical_operators::{ operators::Operator, + physical_planner::{compile, promql_rows, CompiledPhysicalDAG, InputContract, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{compile, promql_rows, CompiledPhysicalDAG, InputContract, Source}, values::{Batch, Value}, }; use common::compile_physical_asap_dag; @@ -44,7 +43,7 @@ fn lower_with(query: &str, accuracy: AccuracyTarget) -> Rc Rc { ..Default::default() }), }; - asap_frontend_promql::unified::lower_promql_workload(&workload, 0) + asap_frontend_promql::lower_promql_workload(&workload, 0) .unwrap() .remove(0) } diff --git a/crates/asap-physical-operators/tests/unified_precompute.rs b/crates/asap-physical-operators/tests/precompute.rs similarity index 92% rename from crates/asap-physical-operators/tests/unified_precompute.rs rename to crates/asap-physical-operators/tests/precompute.rs index 3b6921afc..5435dbb7d 100644 --- a/crates/asap-physical-operators/tests/unified_precompute.rs +++ b/crates/asap-physical-operators/tests/precompute.rs @@ -1,9 +1,6 @@ //! A precompute boundary reads raw samples only from a time series scan. -#[path = "unified_common/mod.rs"] mod common; -use asap_physical_operators::unified_physical_planner::precompute::{ - boundary_schema, raw_sample_schema, -}; +use asap_physical_operators::physical_planner::precompute::{boundary_schema, raw_sample_schema}; use common::compile_physical_asap_dag; use planner_types::ir::{NonASAPOp, Operator, OperatorNode, TimeRangeKind}; use planner_types::pre_asap::{DataType, Field, Schema, Source}; diff --git a/crates/asap-physical-operators/tests/precompute_candidates.rs b/crates/asap-physical-operators/tests/precompute_candidates.rs index 5472d35ec..3b378ad5d 100644 --- a/crates/asap-physical-operators/tests/precompute_candidates.rs +++ b/crates/asap-physical-operators/tests/precompute_candidates.rs @@ -1,16 +1,15 @@ //! Materialized frontiers are compiled by Planner, never rewritten by deployment. -#[path = "unified_common/mod.rs"] mod common; use asap_aware_mapping::{cost_model::DefaultCostModel, search_workload}; use asap_physical_operators::{ factory::create_planner_accumulator, operators::Operator, - runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{ + physical_planner::{ compile, compile_candidate, compile_candidates, cut_candidate, enumerate_frontiers, select_candidate, CandidateCost, CompiledPhysicalDAG, CompiledPhysicalPlan, InputContract, Source, }, + runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; use common::compile_physical_asap_dag; @@ -45,12 +44,11 @@ fn grouped_rate_space() -> asap_aware_mapping::CandidateLogicalASAPDAGs<&'static ..Default::default() }), }; - let root = asap_frontend_promql::unified::lower_promql_workload(&workload, 0) + let root = asap_frontend_promql::lower_promql_workload(&workload, 0) .unwrap() .remove(0); - let root = - asap_physical_operators::unified_physical_planner::promql_rows::with_series_identity(&root) - .unwrap(); + let root = asap_physical_operators::physical_planner::promql_rows::with_series_identity(&root) + .unwrap(); search_workload(vec![("grouped-rate", root)]).unwrap() } @@ -172,7 +170,7 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { let root = dag.roots[0] as u64; let state_id = state.id as u64; let rate_id = evaluation.id as u64; - let frontiers = asap_physical_operators::unified_physical_planner::enumerate_frontiers( + let frontiers = asap_physical_operators::physical_planner::enumerate_frontiers( &dag, &BTreeMap::from([(state_id, InputContract::bounded(input_schema.clone()))]), &[root], @@ -184,7 +182,7 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { assert!(frontiers.contains(&vec![root])); assert!(!frontiers.contains(&vec![root, rate_id])); assert!( - asap_physical_operators::unified_physical_planner::enumerate_frontiers( + asap_physical_operators::physical_planner::enumerate_frontiers( &dag, &BTreeMap::from([(state_id, InputContract::bounded(input_schema.clone()))]), &[root], @@ -257,7 +255,7 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { )]); for frontier in [vec![rate_id, rate_id], vec![root, rate_id], vec![999]] { assert!( - asap_physical_operators::unified_physical_planner::compile_candidate( + asap_physical_operators::physical_planner::compile_candidate( &dag, contracts.clone(), &[root], @@ -380,7 +378,7 @@ fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { /// persistence; an explicit Rate-state input retains its original semantics. #[test] fn bounded_inventory_exposes_grouped_rate_physical_frontiers() { - use asap_physical_operators::unified_physical_planner::enumerate_frontiers; + use asap_physical_operators::physical_planner::enumerate_frontiers; let dag = grouped_rate(); let state = dag .nodes @@ -657,14 +655,12 @@ fn population_topk_cuts_equal_per_frontier_compilation() { ..Default::default() }), }; - let original = asap_frontend_promql::unified::lower_promql_workload(&workload, 0) + let original = asap_frontend_promql::lower_promql_workload(&workload, 0) .unwrap() .remove(0); let root = - asap_physical_operators::unified_physical_planner::promql_rows::with_series_identity( - &original, - ) - .unwrap(); + asap_physical_operators::physical_planner::promql_rows::with_series_identity(&original) + .unwrap(); let selected = asap_aware_mapping::maintained_population::MaintainedPopulationStrategy::new( std::slice::from_ref(&root), ) diff --git a/crates/asap-physical-operators/tests/precompute_population.rs b/crates/asap-physical-operators/tests/precompute_population.rs index 5f15e7ef2..bc25adba5 100644 --- a/crates/asap-physical-operators/tests/precompute_population.rs +++ b/crates/asap-physical-operators/tests/precompute_population.rs @@ -2,8 +2,8 @@ use asap_physical_operators::{ factory::create_planner_accumulator, operators::Operator, + physical_planner::{precompute, CompiledPhysicalDAG, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{precompute, CompiledPhysicalDAG, Source}, values::{Batch, Value}, Statistic, }; diff --git a/crates/asap-physical-operators/tests/promql_binary.rs b/crates/asap-physical-operators/tests/promql_binary.rs index 3aad33e15..882ab35b6 100644 --- a/crates/asap-physical-operators/tests/promql_binary.rs +++ b/crates/asap-physical-operators/tests/promql_binary.rs @@ -1,8 +1,8 @@ //! Binary computation must be fully compiled before deployment binds values. use asap_physical_operators::{ operators::Operator, + physical_planner::{compile_node, CompiledPhysicalDAG, InputContract, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{compile_node, CompiledPhysicalDAG, InputContract, Source}, values::{Batch, SchemaRef, Value}, }; use futures::{executor::block_on, StreamExt}; @@ -163,7 +163,7 @@ fn duplicate_matching_identity_is_rejected() { // Scalar broadcasting and comparison filtering keep the vector operand's value. #[test] fn scalar_broadcast_and_bool_comparison_are_distinct() { - use asap_physical_operators::unified_physical_planner::promql_values; + use asap_physical_operators::physical_planner::promql_values; use planner_types::pre_asap::CompareOpKind; for return_bool in [false, true] { let physical_dag = promql_values::compile_binary( @@ -330,7 +330,7 @@ fn label_map_bool_comparison_drops_the_name() { #[test] fn stored_series_evaluations_support_filters_and_sets() { use asap_physical_operators::{ - summary_kernels::exact::ExactAccumulator, unified_physical_planner::compile, + physical_planner::compile, summary_kernels::exact::ExactAccumulator, }; use planner_types::post_asap::*; use planner_types::pre_asap::{ diff --git a/crates/asap-physical-operators/tests/promql_values.rs b/crates/asap-physical-operators/tests/promql_values.rs index f92662fcd..35b9f2c0b 100644 --- a/crates/asap-physical-operators/tests/promql_values.rs +++ b/crates/asap-physical-operators/tests/promql_values.rs @@ -3,8 +3,8 @@ use asap_physical_operators::expressions::binary::{BinaryOpKind, BinaryOperator} use asap_physical_operators::{ operators::Operator, + physical_planner::{promql_values::*, CompiledPhysicalDAG, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{promql_values::*, CompiledPhysicalDAG, Source}, values::{Batch, Value}, }; use futures::{executor::block_on, StreamExt}; @@ -211,9 +211,9 @@ fn histogram_quantile_keeps_each_label_group() { #[test] fn composed_ensemble_shares_a_producer_across_roots() { use asap_physical_operators::{ + physical_planner::InputContract, plan::{PhysicalOperator, PlanProperties}, runtime::{Input, OutputStream}, - unified_physical_planner::InputContract, values::SchemaRef, }; use planner_types::pre_asap::ArithmeticOpKind; diff --git a/crates/asap-physical-operators/tests/raw_scan.rs b/crates/asap-physical-operators/tests/raw_scan.rs index 2c8585ac7..27921320c 100644 --- a/crates/asap-physical-operators/tests/raw_scan.rs +++ b/crates/asap-physical-operators/tests/raw_scan.rs @@ -368,7 +368,7 @@ fn empty_sources_and_three_valued_predicates() { fn compile_without_readers_and_rebind_inputs() { use asap_physical_operators::{ operators::Operator, - unified_physical_planner::{compile, InputContract, Source}, + physical_planner::{compile, InputContract, Source}, }; let (scan, schema, batches) = fixture(); let dag = plan(scan, &schema, ExecutionDataState::QUERY_ROWS); @@ -403,8 +403,8 @@ fn compile_without_readers_and_rebind_inputs() { #[test] fn compilation_rejects_unknown_boundedness_for_sort() { use asap_physical_operators::{ + physical_planner::{compile, InputContract}, plan::{Boundedness, Emission, PlanProperties}, - unified_physical_planner::{compile, InputContract}, }; let (scan, schema, _) = fixture(); let dag = plan(scan, &schema, ExecutionDataState::QUERY_ROWS); diff --git a/crates/asap-physical-operators/tests/summary_projection.rs b/crates/asap-physical-operators/tests/summary_projection.rs index 2fe0fb864..304bcf8d6 100644 --- a/crates/asap-physical-operators/tests/summary_projection.rs +++ b/crates/asap-physical-operators/tests/summary_projection.rs @@ -3,8 +3,8 @@ use asap_physical_operators::{ expressions::Expression, factory::create_planner_accumulator, operators::Operator, + physical_planner::{compile, CompiledPhysicalDAG, InputContract, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{compile, CompiledPhysicalDAG, InputContract, Source}, values::{Batch, Value}, }; use futures::{executor::block_on, StreamExt}; diff --git a/crates/asap-physical-operators/tests/weighted_topk_binding.rs b/crates/asap-physical-operators/tests/weighted_topk_binding.rs index 6e50c7f61..faa9ab6e4 100644 --- a/crates/asap-physical-operators/tests/weighted_topk_binding.rs +++ b/crates/asap-physical-operators/tests/weighted_topk_binding.rs @@ -1,5 +1,4 @@ //! Planner output binds directly to the shared runtime at a declared rate-value frontier. -#[path = "unified_common/mod.rs"] mod common; use asap_aware_mapping::{ accuracy::{ @@ -193,7 +192,7 @@ fn assert_weighted_binding(evidence: &dyn AccuracyEvidenceProvider, algorithm: S } } -use asap_frontend_promql::unified::lower_promql_workload; +use asap_frontend_promql::lower_promql_workload; use planner_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence as WorkloadEvidence, PlanningWorkload, Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, @@ -202,7 +201,7 @@ use planner_types::workload::{ pub fn lower_promql( query: &str, accuracy: AccuracyTarget, -) -> Result, asap_frontend_promql::unified::PromqlError> { +) -> Result, asap_frontend_promql::PromqlError> { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, @@ -278,7 +277,7 @@ fn direct_rate_topk_preserves_dynamic_unreferenced_labels() { } fn check_direct_rate_topk(dynamic: bool) { - use asap_physical_operators::unified_physical_planner::promql_rows::{ + use asap_physical_operators::physical_planner::promql_rows::{ decode_series_identity, series_row, with_series_identity, SERIES_IDENTITY_COLUMN, }; let mut logical = @@ -336,7 +335,7 @@ fn check_direct_rate_topk(dynamic: bool) { .unwrap_or_else(|| panic!("missing {algorithm:?} over direct Rate")); if dynamic { let (source, ranked) = - asap_physical_operators::unified_physical_planner::promql_rows::compile_rate_ranking( + asap_physical_operators::physical_planner::promql_rows::compile_rate_ranking( candidate, ) .unwrap(); @@ -395,7 +394,7 @@ fn check_direct_rate_topk(dynamic: bool) { .unwrap(); let bytes = serde_json::to_vec(&raw_compiled).unwrap(); let raw_compiled = serde_json::from_slice::< - asap_physical_operators::unified_physical_planner::CompiledPhysicalDAG, + asap_physical_operators::physical_planner::CompiledPhysicalDAG, >(&bytes) .unwrap(); // Each evaluation receives a complete raw window. A reset, a stopped @@ -626,7 +625,7 @@ fn check_direct_rate_topk(dynamic: bool) { // CountSketch; a raw metric does not establish the non-negative CMS contract. #[test] fn spatial_topk_exposes_signed_heap_candidate_over_complete_snapshot() { - use asap_physical_operators::unified_physical_planner::promql_rows::{ + use asap_physical_operators::physical_planner::promql_rows::{ decode_series_identity, series_row, with_series_identity, SERIES_IDENTITY_COLUMN, }; let logical = lower_promql("topk by(job)(1, m)", AccuracyTarget::Epsilon(0.1)).unwrap(); @@ -673,7 +672,7 @@ fn spatial_topk_exposes_signed_heap_candidate_over_complete_snapshot() { ) .unwrap(); let snapshot_program = - asap_physical_operators::unified_physical_planner::promql_rows::compile_current_series_evaluation( + asap_physical_operators::physical_planner::promql_rows::compile_current_series_evaluation( selected, ) .unwrap(); @@ -760,7 +759,7 @@ fn spatial_topk_exposes_signed_heap_candidate_over_complete_snapshot() { // Sort's groups is only the same result when both partition alike. #[test] fn spatial_topk_requires_the_limit_and_sort_partitions_to_agree() { - use asap_physical_operators::unified_physical_planner::promql_rows::with_series_identity; + use asap_physical_operators::physical_planner::promql_rows::with_series_identity; use planner_types::ir::{NonASAPOp, Operator as IrOperator, OperatorNode}; use planner_types::pre_asap::GroupKeys; let series_root = |query: &str| { @@ -910,7 +909,7 @@ fn continuously_maintained_dag(candidate: &Rc) // placement: lifecycle timing, not a separate candidate, puts it in precompute. #[test] fn maintained_rate_heap_lifecycle_compiles_fixed_window_precompute() { - use asap_physical_operators::unified_physical_planner::{ + use asap_physical_operators::physical_planner::{ compile_candidate, promql_rows::with_series_identity, }; let root = Rc::new( @@ -973,38 +972,36 @@ fn maintained_rate_heap_lifecycle_compiles_fixed_window_precompute() { &[heap.id as u64], ) .unwrap(); - let exported = asap_physical_operators::unified_physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&dag).unwrap(); + let exported = asap_physical_operators::physical_planner::promql_rows::compile_fixed_window_rate_aggregation(&dag).unwrap(); assert_eq!( serde_json::to_vec(&exported).unwrap(), serde_json::to_vec(&physical).unwrap() ); // Execute the selected split across a state serialization boundary. // Each run builds fresh weights from that window's counters. - let execute = - |plan: &asap_physical_operators::unified_physical_planner::CompiledPhysicalDAG, - input: Batch, - scope: Scope| { - let id = plan.input_contracts().next().unwrap().0; - let source = - Box::new(Operator::source(input.schema().clone(), vec![input]).unwrap()) - as Source<'static>; - let physical_dag = plan.instantiate(BTreeMap::from([(id, source)])).unwrap(); - block_on(async { - let mut stream = physical_dag - .execute( - plan.roots(), - RunContext::new(scope, Limits::default()).unwrap(), - ) - .unwrap() - .remove(0); - let mut batches = Vec::new(); - while let Some(batch) = stream.next().await { - batches.push((*batch.unwrap()).clone()); - } - assert_eq!(batches.len(), 1); - batches.remove(0) - }) - }; + let execute = |plan: &asap_physical_operators::physical_planner::CompiledPhysicalDAG, + input: Batch, + scope: Scope| { + let id = plan.input_contracts().next().unwrap().0; + let source = Box::new(Operator::source(input.schema().clone(), vec![input]).unwrap()) + as Source<'static>; + let physical_dag = plan.instantiate(BTreeMap::from([(id, source)])).unwrap(); + block_on(async { + let mut stream = physical_dag + .execute( + plan.roots(), + RunContext::new(scope, Limits::default()).unwrap(), + ) + .unwrap() + .remove(0); + let mut batches = Vec::new(); + while let Some(batch) = stream.next().await { + batches.push((*batch.unwrap()).clone()); + } + assert_eq!(batches.len(), 1); + batches.remove(0) + }) + }; let (family, input, grouping) = match &state.payload { PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { family, diff --git a/crates/devtools/examples/canonical_examples.rs b/crates/devtools/examples/canonical_examples.rs index 4cb410e46..fd4b703a1 100644 --- a/crates/devtools/examples/canonical_examples.rs +++ b/crates/devtools/examples/canonical_examples.rs @@ -4,7 +4,7 @@ // plus custom Join/SetOp/Dedup/CTE probes, to eyeball the actual shape. use asap_devtools::lower_promql_with_data_ingestion_interval; -use asap_frontend_sql::unified::{lower_sql_dialect, SqlCatalog}; +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::types::AccuracyTarget; use asap_types::workload::SqlDialect; diff --git a/crates/devtools/src/bin/analyze_corpora.rs b/crates/devtools/src/bin/analyze_corpora.rs index c7fe08b44..0eebe1381 100644 --- a/crates/devtools/src/bin/analyze_corpora.rs +++ b/crates/devtools/src/bin/analyze_corpora.rs @@ -5,7 +5,7 @@ // anomaly report. The default mode remains the ad-hoc SQL/PromQL inspector. use asap_devtools::{lower_promql_with_data_ingestion_interval, SqlCatalog}; -use asap_frontend_sql::unified::lower_sql_dialect; +use asap_frontend_sql::lower_sql_dialect; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::types::AccuracyTarget; use asap_types::workload::SqlDialect; diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index 54dffc17b..053fc8e43 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -27,7 +27,7 @@ use asap_aware_mapping::{explain_replacements, ExplanationKind}; use asap_devtools::lower_promql_with_data_ingestion_interval; -use asap_frontend_sql::unified::{lower_sql_dialect, SqlCatalog}; +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_types::ir::OperatorNode; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::types::AccuracyTarget; diff --git a/crates/devtools/src/bin/variant_coverage.rs b/crates/devtools/src/bin/variant_coverage.rs index a041fe68d..83d797005 100644 --- a/crates/devtools/src/bin/variant_coverage.rs +++ b/crates/devtools/src/bin/variant_coverage.rs @@ -7,7 +7,7 @@ // (`ScalarExpr`) separately. Used to find the minimal IR node set. use asap_devtools::lower_promql_with_data_ingestion_interval; -use asap_frontend_sql::unified::{lower_sql_dialect, SqlCatalog}; +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_types::ir::{OperatorNode, ScalarExpr}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::types::AccuracyTarget; diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index 1e3dea0a8..0316622dc 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -12,12 +12,10 @@ //! only) or [`asap_frontend_sql`] (DataFusion only) — so it never compiles the //! other's parser. -pub use asap_frontend_promql::unified::{ +pub use asap_frontend_promql::{ lower_promql_workload, lower_promql_workload_with_histograms, PromqlError, }; -pub use asap_frontend_sql::unified::{ - lower_sql, lower_sql_batch, SqlCatalog, SqlError, SqlLowerer, -}; +pub use asap_frontend_sql::{lower_sql, lower_sql_batch, SqlCatalog, SqlError, SqlLowerer}; /// Lower one developer-supplied PromQL query with an explicit source cadence. /// Tools intentionally require the cadence rather than choosing a default. diff --git a/crates/frontend-metricsql/src/unified/mod.rs b/crates/frontend-metricsql/src/lib.rs similarity index 100% rename from crates/frontend-metricsql/src/unified/mod.rs rename to crates/frontend-metricsql/src/lib.rs diff --git a/crates/frontend-metricsql/tests/lowering.rs b/crates/frontend-metricsql/tests/lowering.rs index 3add8ee2a..84dd42cea 100644 --- a/crates/frontend-metricsql/tests/lowering.rs +++ b/crates/frontend-metricsql/tests/lowering.rs @@ -1,68 +1,75 @@ +use std::rc::Rc; use std::time::Duration; use asap_frontend_metricsql::{ canonical_metricsql, lower_metricsql, parse_metricsql, MetricsqlError, }; -use asap_types::pre_asap::{AggIntent, QueryExpr, Reduction, Source}; +use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; +use asap_types::pre_asap::{AggIntent, Reduction, Source}; use asap_types::types::AccuracyTarget; -fn lower(query: &str) -> QueryExpr { - lower_metricsql(query, AccuracyTarget::Epsilon(0.01)).unwrap() +/// The workload's ingestion interval, the lookback of a bare selector. +const INTERVAL: Duration = Duration::from_secs(15); + +fn lower(query: &str) -> Rc { + lower_metricsql(query, AccuracyTarget::Epsilon(0.01), INTERVAL).unwrap() } #[test] fn selector_range_aggregate_and_call_share_the_canonical_shape() { let query = r#"sum by (job) (rate(http_requests_total{status=~"5.."}[5m]))"#; - let dag = lower(query); - let QueryExpr::Aggregate { + let tree = lower(query); + let NonASAPOp::Aggregate { reduction, measures, child, .. - } = dag + } = tree.expect_non_asap() else { panic!("expected outer aggregate"); }; - assert_eq!(reduction, Reduction::by(vec![2])); + assert_eq!(reduction, &Reduction::by(vec![2])); assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); - let QueryExpr::Aggregate { + let NonASAPOp::Aggregate { measures, child, .. - } = child.as_ref() + } = child.expect_non_asap() else { panic!("expected rate aggregate"); }; assert!(matches!(measures.as_slice(), [AggIntent::Rate])); - let QueryExpr::TimeRange { range, child } = child.as_ref() else { + let NonASAPOp::TimeRange { range, child, .. } = child.expect_non_asap() else { panic!("expected range"); }; assert_eq!(*range, Duration::from_secs(300)); assert!( - matches!(child.as_ref(), QueryExpr::Scan { source: Source::TimeSeries { metric }, predicates, .. } if metric == "http_requests_total" && predicates.len() == 1) + matches!(child.expect_non_asap(), NonASAPOp::Scan { source: Source::TimeSeries { metric }, predicates, .. } if metric == "http_requests_total" && predicates.len() == 1) ); } #[test] fn default_rollup_with_explicit_range_is_last_over_time() { - let dag = lower("default_rollup(cpu_usage[5m])"); - let QueryExpr::Aggregate { + let tree = lower("default_rollup(cpu_usage[5m])"); + let NonASAPOp::Aggregate { reduction, measures, child, .. - } = dag + } = tree.expect_non_asap() else { panic!("expected aggregate"); }; - assert_eq!(reduction, Reduction::PerEntity); + assert_eq!(reduction, &Reduction::PerEntity); assert!(matches!(measures.as_slice(), [AggIntent::LastOverTime])); assert!( - matches!(child.as_ref(), QueryExpr::TimeRange { range, .. } if *range == Duration::from_secs(300)) + matches!(child.expect_non_asap(), NonASAPOp::TimeRange { range, kind, .. } + if *range == Duration::from_secs(300) && *kind == TimeRangeKind::Range) ); } #[test] fn implicit_default_rollup_requires_runtime_step() { - let error = lower_metricsql("default_rollup(cpu_usage)", AccuracyTarget::Exact).unwrap_err(); + let error = + lower_metricsql("default_rollup(cpu_usage)", AccuracyTarget::Exact, INTERVAL).unwrap_err(); assert!( matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("evaluation step")) ); @@ -75,6 +82,7 @@ fn keep_metric_names_is_preserved_in_ast_and_rejected_without_lineage() { let error = lower_metricsql( "rate(requests_total[5m]) keep_metric_names", AccuracyTarget::Exact, + INTERVAL, ) .unwrap_err(); assert!( @@ -98,7 +106,7 @@ fn victoria_metrics_extension_corpus_parses_natively_and_fails_closed() { let ast = parse_metricsql(query) .unwrap_or_else(|error| panic!("native MetricsQL parser rejected {query:?}: {error}")); assert!(!ast.to_string().is_empty()); - let result = lower_metricsql(query, AccuracyTarget::Exact); + let result = lower_metricsql(query, AccuracyTarget::Exact, INTERVAL); assert!( result.is_err(), "extension semantics must be represented or routed to exact fallback: {query}" @@ -136,7 +144,7 @@ fn canonical_identity_recursively_formats_metricsql_extensions() { fn metricsql_multi_argument_aggregates_fail_closed() { for query in ["sum(foo, bar)", "avg(foo, bar)", "count(foo, bar)"] { parse_metricsql(query).expect("MetricsQL accepts multi-argument aggregates"); - let error = lower_metricsql(query, AccuracyTarget::Exact).unwrap_err(); + let error = lower_metricsql(query, AccuracyTarget::Exact, INTERVAL).unwrap_err(); assert!( matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("requires exactly 1")), "{query} must not silently discard an aggregate input" @@ -147,7 +155,13 @@ fn metricsql_multi_argument_aggregates_fail_closed() { #[test] fn supported_parameterized_functions_require_their_exact_arity() { let quantile = lower("quantile(0.9, requests_total)"); - assert!(matches!(quantile, QueryExpr::Aggregate { .. })); + assert!(matches!( + quantile.expect_non_asap(), + NonASAPOp::Aggregate { .. } + )); let rollup = lower("quantile_over_time(0.9, requests_total[5m])"); - assert!(matches!(rollup, QueryExpr::Aggregate { .. })); + assert!(matches!( + rollup.expect_non_asap(), + NonASAPOp::Aggregate { .. } + )); } diff --git a/crates/frontend-metricsql/tests/unified_lowering.rs b/crates/frontend-metricsql/tests/lowering_semantics.rs similarity index 94% rename from crates/frontend-metricsql/tests/unified_lowering.rs rename to crates/frontend-metricsql/tests/lowering_semantics.rs index a5e9d2b42..ea24bf1e4 100644 --- a/crates/frontend-metricsql/tests/unified_lowering.rs +++ b/crates/frontend-metricsql/tests/lowering_semantics.rs @@ -1,7 +1,7 @@ -//! The unified MetricsQL lowering (`asap_frontend_metricsql::unified`). +//! MetricsQL lowering semantics: counts, rollup ranges, instant lookback. use std::time::Duration; -use asap_frontend_metricsql::unified::{lower_metricsql_query_workload, MetricsqlError}; +use asap_frontend_metricsql::{lower_metricsql_query_workload, MetricsqlError}; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::pre_asap::AggIntent; use asap_types::types::AccuracyTarget; @@ -16,7 +16,7 @@ fn lower_metricsql( query: &str, accuracy: AccuracyTarget, ) -> Result, MetricsqlError> { - asap_frontend_metricsql::unified::lower_metricsql(query, accuracy, INTERVAL) + asap_frontend_metricsql::lower_metricsql(query, accuracy, INTERVAL) } fn workload(language: QueryLanguage, query: &str) -> PlanningWorkload { diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index b8576ae9b..569c87c9a 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -3,8 +3,9 @@ name = "asap-frontend-promql" version = "0.1.0" edition = "2021" -# PromQL front end: L1 (parse) → L2 relational, then the shared L2→L3 converter -# — both in asap-types. Pulls the PromQL parser only — never DataFusion. +# PromQL front end: parse → the shared name-based `UnresolvedOp` tree +# (asap-frontend-common) → the unified IR. Pulls the PromQL parser only — +# never DataFusion. [dependencies] asap-types = { path = "../types" } asap-frontend-common = { path = "../frontend-common" } diff --git a/crates/frontend-promql/src/unified/error.rs b/crates/frontend-promql/src/error.rs similarity index 97% rename from crates/frontend-promql/src/unified/error.rs rename to crates/frontend-promql/src/error.rs index a889d6710..ebbcf49c2 100644 --- a/crates/frontend-promql/src/unified/error.rs +++ b/crates/frontend-promql/src/error.rs @@ -31,7 +31,7 @@ pub enum PromqlError { InvalidParameter(String), /// The workload's query language is not PromQL. WrongLanguage(String), - /// Resolving the canonical unresolved tree failed (name resolution + /// Resolving the canonical unresolved DAG failed (name resolution /// against the bound schema). Convert(ResolveDAGError), } diff --git a/crates/frontend-promql/src/unified/histogram.rs b/crates/frontend-promql/src/histogram.rs similarity index 100% rename from crates/frontend-promql/src/unified/histogram.rs rename to crates/frontend-promql/src/histogram.rs diff --git a/crates/frontend-promql/src/unified/mod.rs b/crates/frontend-promql/src/lib.rs similarity index 100% rename from crates/frontend-promql/src/unified/mod.rs rename to crates/frontend-promql/src/lib.rs diff --git a/crates/frontend-promql/src/unified/promql.rs b/crates/frontend-promql/src/promql.rs similarity index 99% rename from crates/frontend-promql/src/unified/promql.rs rename to crates/frontend-promql/src/promql.rs index 01430a00e..0e8b3df1c 100644 --- a/crates/frontend-promql/src/unified/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -16,7 +16,7 @@ //! the schema-*dependent* work: binding every `ColumnRef` to its //! positional `ColumnId`. //! -//! # PromQL → canonical unresolved-tree mapping (summary) +//! # PromQL → canonical unresolved-DAG mapping (summary) //! //! | PromQL | Canonical shape | //! |---|---| @@ -84,11 +84,11 @@ use asap_types::types::AccuracyTarget; /// Every scalar expression this front end builds follows PromQL's numeric rules. const PROMQL: ExprSemantics = ExprSemantics::Promql; -use crate::unified::error::PromqlError as LoweringError; +use crate::error::PromqlError as LoweringError; type Result = std::result::Result; -/// Parses and lowers (→ the canonical, unresolved tree) a PromQL query string. +/// Parses and lowers (→ the canonical, unresolved DAG) a PromQL query string. pub(crate) struct PromqlLowerer; #[derive(Debug, Clone)] @@ -193,8 +193,8 @@ impl PromqlLowerer { let mut metrics = Vec::new(); collect_metric_names(&ast, &mut metrics); if metrics.iter().any(|metric| { - crate::unified::histogram::current_kind_of(metric) - == Some(crate::unified::histogram::HistogramKind::Native) + crate::histogram::current_kind_of(metric) + == Some(crate::histogram::HistogramKind::Native) }) { return Err(LoweringError::UnsupportedFeature( "native histogram samples have no IR representation".into(), @@ -477,8 +477,8 @@ fn vector_binary( /// `extract_matrix` can't accept it. Lower the sub-query recursively and reduce /// it per series (issue #27). fn walk_call(call: &Call) -> Result { - if let Some(tree) = range_fn_over_subquery(call)? { - return Ok(tree); + if let Some(dag) = range_fn_over_subquery(call)? { + return Ok(dag); } build(lower_inner_call(call)?, vec![], Outer::None) } @@ -721,11 +721,11 @@ fn outer_kind(agg: &AggregateExpr) -> Result { /// `Reduce(without(keys))`: a `without` grouping is never label-preserving — /// per-entity requires `!by.is_without()` — so this both re-tags an existing /// `Reduce` and upgrades a wrongly-early `PerEntity` guess, uniformly. -fn mark_without(tree: Unresolved, without: bool) -> Unresolved { +fn mark_without(dag: Unresolved, without: bool) -> Unresolved { if !without { - return tree; + return dag; } - match tree { + match dag { Unresolved::Aggregate { reduction, measures, @@ -1058,7 +1058,7 @@ fn is_presence_fn(name: &str) -> bool { /// `absent(v)` / `absent_over_time(m[w])` / `present_over_time(m[w])` — lowered /// to an `Aggregate{[Absent/…]}` over the (instant or range) argument. The /// empty-result → synthesized-1-sample logic is a post-ASAP/runtime concern; -/// the canonical tree only marks the operation (issue #47). +/// the canonical DAG only marks the operation (issue #47). fn walk_presence(call: &Call) -> Result { let func = match call.func.name { "absent" => AggIntent::Absent, @@ -1325,7 +1325,7 @@ fn clamp_bounds_filter(min: &Scalar, max: &Scalar, child: Unresolved) -> Unresol /// `Quantile` (`true`) or exact classic-bucket interpolation (`false`). /// /// Metadata wins: if any metric referenced in `arg` has a declared -/// [`HistogramKind`](crate::unified::histogram::HistogramKind), that decides it (issue +/// [`HistogramKind`](crate::histogram::HistogramKind), that decides it (issue /// #79) — this fixes both the false-positive (a `…_bucket`-named non-histogram /// declared `RawSamples`) and the false-negative (a suffix-less classic /// histogram declared `ClassicBucket`) of the structural heuristic. With no @@ -1335,9 +1335,9 @@ fn histogram_arg_is_sketchable(arg: &Expr) -> Result { collect_metric_names(arg, &mut metrics); let kinds = metrics .iter() - .filter_map(|metric| crate::unified::histogram::current_kind_of(metric)) + .filter_map(|metric| crate::histogram::current_kind_of(metric)) .collect::>(); - if kinds.contains(&crate::unified::histogram::HistogramKind::Native) { + if kinds.contains(&crate::histogram::HistogramKind::Native) { return Err(LoweringError::UnsupportedFeature( "native histogram samples have no IR representation".into(), )); @@ -1624,7 +1624,7 @@ fn lower_inner_call(call: &Call) -> Result { } } -/// Assemble the Layer-2 tree from a lowered inner vector, the resolved group +/// Assemble the Layer-2 DAG from a lowered inner vector, the resolved group /// keys, and the enclosing aggregator shape. fn build(inner: Inner, keys: Vec, outer: Outer) -> Result { match outer { @@ -1712,7 +1712,7 @@ fn build(inner: Inner, keys: Vec, outer: Outer) -> Result }; let additive_ranking = measure.is_supported(descending); if additive_ranking { - // Preserve the ranked aggregate intent in the canonical tree so the + // Preserve the ranked aggregate intent in the canonical DAG so the // intent algebra is explicit about what is being computed. // Post-ASAP binding may fuse the Count and TopK into a // single-pass heavy-hitter sketch (SpaceSaving / @@ -1763,7 +1763,7 @@ fn build(inner: Inner, keys: Vec, outer: Outer) -> Result /// Decide `PerEntity` vs `Reduce(by)` for a canonical `Aggregate`, entirely /// from local PromQL semantics: the keys and whether this operation preserves -/// each input series. It never infers entity reduction from the child tree's +/// each input series. It never infers entity reduction from the child DAG's /// temporal shape. `without()` is applied /// separately, post-hoc, by `mark_without` — see its doc for why that's still /// correct here. diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index b401d1e5c..dc0b6d001 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -5,7 +5,6 @@ use asap_aware_mapping::{ default_strategies, search_workload_with_targets, ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG, }; -#[path = "unified_support.rs"] mod support; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::{ASAPOp, Operator}; diff --git a/crates/frontend-promql/tests/unified_histogram_metadata.rs b/crates/frontend-promql/tests/histogram_metadata.rs similarity index 96% rename from crates/frontend-promql/tests/unified_histogram_metadata.rs rename to crates/frontend-promql/tests/histogram_metadata.rs index 49fdffe02..aba373804 100644 --- a/crates/frontend-promql/tests/unified_histogram_metadata.rs +++ b/crates/frontend-promql/tests/histogram_metadata.rs @@ -5,15 +5,14 @@ //! heuristic's false-positive and false-negative cases. Undeclared metrics still //! fall back to the heuristic. -use asap_frontend_promql::unified::{HistogramCatalog, HistogramKind}; -#[path = "unified_support.rs"] +use asap_frontend_promql::{HistogramCatalog, HistogramKind}; mod support; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::pre_asap::AggIntent; use asap_types::types::AccuracyTarget; use support::{lower_promql, lower_promql_with_histograms}; -/// The histogram/quantile intent kind in the lowered tree: `"HQ"` for the +/// The histogram/quantile intent kind in the lowered DAG: `"HQ"` for the /// classic-bucket `HistogramQuantile`, `"Q"` for the sketch-able `Quantile`. fn quantile_kind(qe: &OperatorNode) -> &'static str { fn walk(e: &OperatorNode) -> Option<&'static str> { diff --git a/crates/frontend-promql/tests/maintained_population_horizon.rs b/crates/frontend-promql/tests/maintained_population_horizon.rs index ddd3c2f63..b4130bf86 100644 --- a/crates/frontend-promql/tests/maintained_population_horizon.rs +++ b/crates/frontend-promql/tests/maintained_population_horizon.rs @@ -1,4 +1,3 @@ -#[path = "unified_support.rs"] mod support; use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; diff --git a/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs b/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs index aa74a5b18..5e78e261d 100644 --- a/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs +++ b/crates/frontend-promql/tests/observability/awesome_prometheus_alerts.rs @@ -26,8 +26,8 @@ // `__GAP`-suffixed names intentionally SHOUT the documented divergences. #![allow(non_snake_case)] -use asap_frontend_promql::unified::PromqlError as LoweringError; -#[path = "../unified_support.rs"] +use asap_frontend_promql::PromqlError as LoweringError; +#[path = "../support.rs"] mod support; use std::rc::Rc; diff --git a/crates/frontend-promql/tests/observability/metrics_observability.rs b/crates/frontend-promql/tests/observability/metrics_observability.rs index 88a51f65c..d40df6236 100644 --- a/crates/frontend-promql/tests/observability/metrics_observability.rs +++ b/crates/frontend-promql/tests/observability/metrics_observability.rs @@ -10,8 +10,8 @@ use asap_aware_mapping::replacement::{retain_exact, RealizationError}; use asap_aware_mapping::{ ASAPStrategies, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_frontend_promql::unified::PromqlError; -#[path = "../unified_support.rs"] +use asap_frontend_promql::PromqlError; +#[path = "../support.rs"] mod support; use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index 6b76f5755..45afdeb5f 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -19,8 +19,8 @@ use asap_aware_mapping::replacement::{retain_exact, RealizationError}; use asap_aware_mapping::{ ASAPStrategies, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_frontend_promql::unified::PromqlError as LoweringError; -#[path = "../unified_support.rs"] +use asap_frontend_promql::PromqlError as LoweringError; +#[path = "../support.rs"] mod support; use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; @@ -75,7 +75,7 @@ impl Tally { fn tally(corpus: &str) -> Tally { let mut t = Tally::default(); for q in queries(corpus) { - match asap_frontend_promql::unified::lower_promql_query_workload( + match asap_frontend_promql::lower_promql_query_workload( &support::workload(q, AccuracyTarget::Exact), 0, ) { diff --git a/crates/frontend-promql/tests/promql_binding_regressions.rs b/crates/frontend-promql/tests/promql_binding_regressions.rs index ba006ae4a..416b6680d 100644 --- a/crates/frontend-promql/tests/promql_binding_regressions.rs +++ b/crates/frontend-promql/tests/promql_binding_regressions.rs @@ -1,6 +1,5 @@ use asap_types::types::AccuracyTarget; -#[path = "unified_support.rs"] mod support; use support::lower_promql; diff --git a/crates/frontend-promql/tests/unified_promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs similarity index 99% rename from crates/frontend-promql/tests/unified_promql_conformance.rs rename to crates/frontend-promql/tests/promql_conformance.rs index 3f7f4b940..026468734 100644 --- a/crates/frontend-promql/tests/unified_promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -1,8 +1,8 @@ -//! PromQL **semantic conformance** for the parse-to-canonical-tree lowering. +//! PromQL **semantic conformance** for the parse-to-canonical-DAG lowering. //! //! We *lower* PromQL to the intent algebra; we do not *execute* it. So "same //! semantic job as Prometheus" here means: for each canonical query, does the -//! canonical tree encode the **documented PromQL meaning** — and where we knowingly +//! canonical DAG encode the **documented PromQL meaning** — and where we knowingly //! diverge (reject, approximate, or drop a modifier), is that pinned by a test //! so it stays visible? //! @@ -34,8 +34,7 @@ use std::rc::Rc; use std::time::Duration; -use asap_frontend_promql::unified::PromqlError as LoweringError; -#[path = "unified_support.rs"] +use asap_frontend_promql::PromqlError as LoweringError; mod support; use asap_types::ir::{ BinaryOperator, ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind, @@ -60,7 +59,7 @@ fn ok(q: &str) -> Rc { fn rejected(q: &str) -> LoweringError { match lower_promql(q, AccuracyTarget::Exact) { Err(e) => e, - Ok(tree) => panic!("expected {q:?} to be rejected, but it lowered to: {tree:?}"), + Ok(dag) => panic!("expected {q:?} to be rejected, but it lowered to: {dag:?}"), } } @@ -189,7 +188,7 @@ fn name_regex_matcher_is_rejected__GAP() { #[test] fn range_vector_selector_is_time_range() { // SEMANTICS: `[5m]` turns an instant vector into a range vector, - // represented in the canonical tree as a dedicated `TimeRange` node. + // represented in the canonical DAG as a dedicated `TimeRange` node. let qe = ok("node_cpu_seconds_total[5m]"); let NonASAPOp::TimeRange { range, .. } = qe.expect_non_asap() else { panic!("expected TimeRange for a range-vector selector, got {qe:?}"); @@ -527,7 +526,7 @@ fn histogram_quantile_over_rate() { fn histogram_quantile_over_sum_by_le_preserves_le_grouping() { // SEMANTICS: the standard pattern — bucket rates summed by `le`, then the // quantile. The `sum by (le)` aggregation must survive into the - // canonical tree. + // canonical DAG. let qe = ok( "histogram_quantile(0.99, sum by(le) (rate(demo_api_request_duration_seconds_bucket[5m])))", ); @@ -818,7 +817,7 @@ fn outer_aggregate_over_nested_aggregate_nests() { // `max(sum by (job) (rate(m[5m])))` — an outer cross-series reduction over a // nested per-group reduction over a per-series rate: three stacked levels the // flat two-level template rejected. Each level survives into the - // canonical tree (issue #27). + // canonical DAG (issue #27). let qe = ok("max(sum by (job) (rate(http_requests_total[5m])))"); let NonASAPOp::Aggregate { measures, child, .. @@ -914,7 +913,7 @@ fn outer_group_key_over_binary_op_resolves_on_both_sides() { // Issue #52: an outer aggregate's group key that appears in *neither* side of // a binary op — the metric-name label `__name__`, or a plain `job` — must // still resolve. Each `or` side is bound independently against its own - // sub-tree, so the key is seeded as an inherited column on both sides. + // sub-DAG, so the key is seeded as an inherited column on both sides. let qe = ok(r#"sum by (__name__)(metric_a{env="1"} or metric_b{env="2"})"#); let NonASAPOp::Aggregate { reduction, child, .. @@ -1765,7 +1764,7 @@ fn timestamp_of_a_non_selector_is_the_eval_time() { #[test] fn timestamp_composes_under_an_outer_aggregation() { // `sum by (job) (timestamp(up))` — the per-series `timestamp` transform sits - // below an ordinary grouped sum. Both intents must appear in the tree. + // below an ordinary grouped sum. Both intents must appear in the DAG. let qe = ok("sum by (job) (timestamp(up))"); assert!(has(&qe, |i| *i == AggIntent::TimeFn(TimeFunc::Timestamp))); assert!(has(&qe, |i| matches!(i, AggIntent::Sum { .. }))); diff --git a/crates/frontend-promql/tests/promql_equivalence.rs b/crates/frontend-promql/tests/promql_equivalence.rs index 230cf2a05..ac3c5a06f 100644 --- a/crates/frontend-promql/tests/promql_equivalence.rs +++ b/crates/frontend-promql/tests/promql_equivalence.rs @@ -19,7 +19,6 @@ use std::rc::Rc; -#[path = "unified_support.rs"] mod support; use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; diff --git a/crates/frontend-promql/tests/unified_promql_lowering.rs b/crates/frontend-promql/tests/promql_lowering.rs similarity index 98% rename from crates/frontend-promql/tests/unified_promql_lowering.rs rename to crates/frontend-promql/tests/promql_lowering.rs index 6097b1ec6..1aee42cdc 100644 --- a/crates/frontend-promql/tests/unified_promql_lowering.rs +++ b/crates/frontend-promql/tests/promql_lowering.rs @@ -1,4 +1,4 @@ -//! End-to-end tests for PromQL → unresolved → canonical tree lowering. +//! End-to-end tests for PromQL → unresolved → canonical DAG lowering. use std::rc::Rc; use std::time::Duration; @@ -15,8 +15,7 @@ use asap_types::workload::{ Predictability, Query, QueryLanguage, QueryRequirements, QueryWorkload, TimeSelection, }; -use asap_frontend_promql::unified::{lower_promql_workload, PromqlError as LoweringError}; -#[path = "unified_support.rs"] +use asap_frontend_promql::{lower_promql_workload, PromqlError as LoweringError}; mod support; use support::lower_promql; @@ -60,14 +59,14 @@ fn distinct_over_time_preserves_cardinality_accuracy_and_nested_windows() { "sum by(job)(distinct_over_time(cpu_usage[5m]))", ] { for accuracy in [AccuracyTarget::Exact, AccuracyTarget::Epsilon(0.02)] { - let tree = lower_promql(query, accuracy.clone()).unwrap(); + let dag = lower_promql(query, accuracy.clone()).unwrap(); let mut intents = Vec::new(); - collect_intents(&tree, &mut intents); + collect_intents(&dag, &mut intents); assert!( intents.iter().any(|intent| matches!( intent, AggIntent::Cardinality { accuracy: actual, .. } if actual == &accuracy )), - "{query}: {tree:?}" + "{query}: {dag:?}" ); assert!(!intents .iter() @@ -283,7 +282,7 @@ fn histogram_quantile_over_sum_by_le_preserves_grouping() { // The canonical Prometheus histogram pattern. Previously returned // UnsupportedFeature because `extract_matrix` couldn't see through the // `sum by (le)` aggregate; now the `le` grouping survives into the - // canonical tree. + // canonical DAG. let qe = lower(r#"histogram_quantile(0.99, sum by (le) (rate(http_requests_bucket[5m])))"#); let NonASAPOp::Aggregate { measures, child, .. @@ -504,19 +503,19 @@ fn count_over_distinct_over_time_preserves_both_aggregates() { Reduction::by(vec![2]), ), ] { - let tree = lower(query); + let dag = lower(query); let NonASAPOp::Aggregate { measures, reduction: actual, child, .. - } = tree.expect_non_asap() + } = dag.expect_non_asap() else { - panic!("expected outer Count: {tree:?}"); + panic!("expected outer Count: {dag:?}"); }; assert!( matches!(measures.as_slice(), [AggIntent::Count { .. }]), - "{query}: {tree:?}" + "{query}: {dag:?}" ); assert_eq!(actual, &reduction, "{query}"); let NonASAPOp::Aggregate { @@ -526,11 +525,11 @@ fn count_over_distinct_over_time_preserves_both_aggregates() { .. } = child.expect_non_asap() else { - panic!("expected inner per-series Cardinality: {tree:?}"); + panic!("expected inner per-series Cardinality: {dag:?}"); }; assert!( matches!(measures.as_slice(), [AggIntent::Cardinality { .. }]), - "{query}: {tree:?}" + "{query}: {dag:?}" ); assert_eq!(reduction, &Reduction::PerEntity, "{query}"); assert!( @@ -552,17 +551,17 @@ fn count_never_lowers_to_distinct_sample_values() { "count(count_over_time(up[5m]))", "count_over_time(up[5m])", ] { - let tree = lower(query); - let intents = all_intents(&tree); + let dag = lower(query); + let intents = all_intents(&dag); assert!( intents.iter().any(|i| matches!(i, AggIntent::Count { .. })), - "{query}: {tree:?}" + "{query}: {dag:?}" ); assert!( !intents .iter() .any(|i| matches!(i, AggIntent::Cardinality { .. })), - "{query}: {tree:?}" + "{query}: {dag:?}" ); } } @@ -901,7 +900,7 @@ fn binary_op_binds_each_branch_against_its_own_schema() { ); } -/// Collect every `AggIntent` in the tree, root-to-leaf. +/// Collect every `AggIntent` in the dag, root-to-leaf. fn all_intents(e: &OperatorNode) -> Vec { let mut out = Vec::new(); collect_intents(e, &mut out); @@ -928,7 +927,7 @@ fn collect_intents(e: &OperatorNode, out: &mut Vec) { } } -/// True if any `AggIntent` anywhere in the tree satisfies `pred`. +/// True if any `AggIntent` anywhere in the dag satisfies `pred`. fn has_intent bool>(e: &OperatorNode, pred: F) -> bool { all_intents(e).iter().any(pred) } @@ -1502,7 +1501,7 @@ fn explicit_range_selector_is_a_range_time_range() { #[test] fn instant_and_range_selectors_of_equal_length_stay_distinct() { - // The kind is part of the shape: a 1s range selector is not the same tree as + // The kind is part of the shape: a 1s range selector is not the same dag as // the 1s instant lookback injected around a bare selector. assert_ne!(lower("up"), lower("up[1s]")); } diff --git a/crates/frontend-promql/tests/unified_scalar_design.rs b/crates/frontend-promql/tests/scalar_design.rs similarity index 98% rename from crates/frontend-promql/tests/unified_scalar_design.rs rename to crates/frontend-promql/tests/scalar_design.rs index aca48e40e..09fa3d9b6 100644 --- a/crates/frontend-promql/tests/unified_scalar_design.rs +++ b/crates/frontend-promql/tests/scalar_design.rs @@ -1,12 +1,11 @@ //! Scalar expressions never become constant-wrapper operators. -#[path = "unified_support.rs"] mod support; use asap_types::ir::{NonASAPOp, QueryRoot, ScalarExpr}; use asap_types::pre_asap::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::types::AccuracyTarget; fn root(query: &str) -> QueryRoot { - asap_frontend_promql::unified::lower_promql_query_workload( + asap_frontend_promql::lower_promql_query_workload( &support::workload(query, AccuracyTarget::Exact), 0, ) diff --git a/crates/frontend-promql/tests/unified_support.rs b/crates/frontend-promql/tests/support.rs similarity index 97% rename from crates/frontend-promql/tests/unified_support.rs rename to crates/frontend-promql/tests/support.rs index 5ed0127b9..1bd247d04 100644 --- a/crates/frontend-promql/tests/unified_support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -1,6 +1,6 @@ use std::rc::Rc; -use asap_frontend_promql::unified::{ +use asap_frontend_promql::{ lower_promql_workload, lower_promql_workload_with_histograms, HistogramCatalog, PromqlError, }; use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; @@ -87,7 +87,7 @@ pub fn post_asap_dag(root: &Rc) -> asap_types::ir::physical_export #[allow(dead_code)] pub fn scalar_root(query: &str) -> ScalarExpr { - match asap_frontend_promql::unified::lower_promql_query_workload( + match asap_frontend_promql::lower_promql_query_workload( &workload(query, AccuracyTarget::Exact), 0, ) diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs index eaf12e2e2..f19429e67 100644 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ b/crates/frontend-promql/tests/univmon_candidates.rs @@ -6,7 +6,6 @@ use asap_aware_mapping::accuracy::{ use asap_aware_mapping::cost_model::DefaultCostModel; use asap_aware_mapping::replacement::{default_strategies, search_workload_with_targets}; use asap_aware_mapping::{ASAPStrategies, Replacement, ReplacementStrategy, TargetSubDAG}; -#[path = "unified_support.rs"] mod support; use asap_types::ir::cse::share_common_sub_dags; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; diff --git a/crates/frontend-sql/src/unified/error.rs b/crates/frontend-sql/src/error.rs similarity index 100% rename from crates/frontend-sql/src/unified/error.rs rename to crates/frontend-sql/src/error.rs diff --git a/crates/frontend-sql/src/unified/mod.rs b/crates/frontend-sql/src/lib.rs similarity index 98% rename from crates/frontend-sql/src/unified/mod.rs rename to crates/frontend-sql/src/lib.rs index 279585fb9..1747b09c8 100644 --- a/crates/frontend-sql/src/unified/mod.rs +++ b/crates/frontend-sql/src/lib.rs @@ -26,7 +26,7 @@ pub use sql::{SqlCatalog, SqlLowerer}; /// /// The `catalog` supplies table schemas (used both to plan the SQL with /// DataFusion and to carry positional column identity into the resolved -/// tree). `accuracy` is threaded onto every approximate intent as it's built. +/// DAG). `accuracy` is threaded onto every approximate intent as it's built. pub async fn lower_sql( query: &str, catalog: &SqlCatalog, diff --git a/crates/frontend-sql/src/unified/sql/clickhouse_ast.rs b/crates/frontend-sql/src/sql/clickhouse_ast.rs similarity index 100% rename from crates/frontend-sql/src/unified/sql/clickhouse_ast.rs rename to crates/frontend-sql/src/sql/clickhouse_ast.rs diff --git a/crates/frontend-sql/src/unified/sql/collection_planning.rs b/crates/frontend-sql/src/sql/collection_planning.rs similarity index 99% rename from crates/frontend-sql/src/unified/sql/collection_planning.rs rename to crates/frontend-sql/src/sql/collection_planning.rs index 09b0bfa8e..62bbbbb6d 100644 --- a/crates/frontend-sql/src/unified/sql/collection_planning.rs +++ b/crates/frontend-sql/src/sql/collection_planning.rs @@ -162,7 +162,7 @@ impl ScalarUDFImpl for CollectionPlanningFunction { Ok(Arc::new(ArrowField::new(self.name, dtype, nullable))) } fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { - Err(DataFusionError::NotImplemented("collection planning adapter cannot execute; use a capable query engine or external exact sub_dag".into())) + Err(DataFusionError::NotImplemented("collection planning adapter cannot execute; use a capable query engine or external exact sub-DAG".into())) } } diff --git a/crates/frontend-sql/src/unified/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs similarity index 99% rename from crates/frontend-sql/src/unified/sql/expr.rs rename to crates/frontend-sql/src/sql/expr.rs index f58af5615..1ff5dcc51 100644 --- a/crates/frontend-sql/src/unified/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -6,7 +6,7 @@ use asap_frontend_common::UnresolvedScalar as Unresolved; use asap_types::ir::ExprSemantics; use asap_types::pre_asap::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; -use crate::unified::error::SqlError as LoweringError; +use crate::error::SqlError as LoweringError; use super::types::{cast_target, scalar_value_to_asap}; use super::SqlLowerer; @@ -329,7 +329,7 @@ pub(super) fn split_disjuncts(expr: &Expr) -> Vec<&Expr> { #[cfg(test)] mod tests { use super::*; - use crate::unified::sql::SqlCatalog; + use crate::sql::SqlCatalog; use asap_types::pre_asap::schema::DataType; use datafusion::common::ScalarValue as DfScalarValue; diff --git a/crates/frontend-sql/src/unified/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs similarity index 99% rename from crates/frontend-sql/src/unified/sql/mod.rs rename to crates/frontend-sql/src/sql/mod.rs index b5c898779..ad0f1c41e 100644 --- a/crates/frontend-sql/src/unified/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -72,7 +72,7 @@ use asap_types::pre_asap::{ use asap_types::types::AccuracyTarget; use asap_types::workload::SqlDialect; -use crate::unified::error::SqlError as LoweringError; +use crate::error::SqlError as LoweringError; mod clickhouse_ast; mod collection_planning; @@ -2143,7 +2143,7 @@ struct DerivedCols<'l> { lowerer: &'l SqlLowerer<'l>, cols: Vec, /// Whether any column is genuinely derived. Without one the aggregate keeps - /// its original child, so trees that lower today keep their exact shape. + /// its original child, so DAGs that lower today keep their exact shape. any: bool, /// First same-name-different-value collision, reported only if the /// projection is actually inserted (see [`Self::wrap`]). @@ -2276,7 +2276,7 @@ impl<'l> DerivedCols<'l> { /// Wrap `input` in the materializing `Project`, or return it untouched when /// nothing needed deriving — so a query that lowers today keeps its exact - /// tree, and a name collision that the projection would have flattened only + /// DAG, and a name collision that the projection would have flattened only /// matters once the projection exists. fn wrap(self, input: Unresolved) -> Result { if !self.any { diff --git a/crates/frontend-sql/src/unified/sql/types.rs b/crates/frontend-sql/src/sql/types.rs similarity index 99% rename from crates/frontend-sql/src/unified/sql/types.rs rename to crates/frontend-sql/src/sql/types.rs index 61c9d8cb7..eeb56ff1f 100644 --- a/crates/frontend-sql/src/unified/sql/types.rs +++ b/crates/frontend-sql/src/sql/types.rs @@ -1,6 +1,6 @@ //! Type bridges between DataFusion's Arrow types and the canonical `DataType`, plus //! the SQL table catalog used to register tables with DataFusion and to carry -//! resolved leaf schemas into the canonical, unresolved tree. +//! resolved leaf schemas into the canonical, unresolved DAG. use std::collections::HashMap; @@ -12,7 +12,7 @@ use datafusion::common::ScalarValue as DfScalarValue; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::pre_asap::ScalarValue; -use crate::unified::error::SqlError as LoweringError; +use crate::error::SqlError as LoweringError; /// Table catalog for SQL lowering: table name → resolved canonical [`Schema`]. /// diff --git a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs index bd985be98..cc24329cd 100644 --- a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs +++ b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs @@ -37,7 +37,7 @@ use std::rc::Rc; -use asap_frontend_sql::unified::{lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::pre_asap::{AggIntent, GroupKeys}; diff --git a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs index 795c2da76..d7c750d7d 100644 --- a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs +++ b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs @@ -13,7 +13,7 @@ //! per-query), this corpus is 200 queries wide and only pins an **aggregate //! tally** by outcome category -- see the module doc on [`Category`] for why. -use asap_frontend_sql::unified::{lower_sql_dialect, SqlCatalog, SqlError}; +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::types::AccuracyTarget; use asap_types::workload::SqlDialect; diff --git a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs index 3469642e9..cce394a08 100644 --- a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs +++ b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs @@ -21,7 +21,7 @@ use std::rc::Rc; -use asap_frontend_sql::unified::{lower_sql, SqlCatalog, SqlError as LoweringError}; +use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::pre_asap::{AggIntent, GroupKeys}; diff --git a/crates/frontend-sql/tests/maintained_population.rs b/crates/frontend-sql/tests/maintained_population.rs index 4c7e97c9d..dc7d55c44 100644 --- a/crates/frontend-sql/tests/maintained_population.rs +++ b/crates/frontend-sql/tests/maintained_population.rs @@ -1,6 +1,6 @@ //! SQL and PromQL use the same shared-state rule without sharing membership semantics. use asap_aware_mapping::maintained_population::MaintainedPopulationStrategy; -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::{ ir::{ apply_lifecycle_timings, cse::share_common_sub_dags, diff --git a/crates/frontend-sql/tests/netflow/netflow.rs b/crates/frontend-sql/tests/netflow/netflow.rs index 72c05154c..da680d664 100644 --- a/crates/frontend-sql/tests/netflow/netflow.rs +++ b/crates/frontend-sql/tests/netflow/netflow.rs @@ -6,7 +6,7 @@ use std::rc::Rc; -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::pre_asap::{AggIntent, GroupKeys}; diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 425e0708a..d4930c98d 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -1,7 +1,7 @@ //! Correlation retains both arguments through SQL lowering and exact planning. use std::rc::Rc; -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::{ apply_lifecycle_timings, physical_export::compile_physical_asap_dag, LifecycleAssignment, NonASAPOp, OperatorNode, ScalarExpr, TimingMemo, diff --git a/crates/frontend-sql/tests/unified_sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs similarity index 99% rename from crates/frontend-sql/tests/unified_sql_lowering.rs rename to crates/frontend-sql/tests/sql_lowering.rs index 2a99c5660..9444f835b 100644 --- a/crates/frontend-sql/tests/unified_sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -7,7 +7,6 @@ //! resolution, so a successful `lower` already proves schema derivation is //! total over the tree. -use ::asap_frontend_sql::unified as asap_frontend_sql; use asap_types::ir::Predicate; use std::rc::Rc; @@ -232,7 +231,7 @@ async fn where_folds_predicate_onto_scan() { async fn multi_aggregate_group_by_binds_columns_positionally() { // SUM(bytes)=col 3, AVG(latency)=col 2, GROUP BY service=col 1. let qe = lower("SELECT service, SUM(bytes), AVG(latency) FROM metrics GROUP BY service").await; - let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate in the tree"); + let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate in the DAG"); assert_eq!(by, &vec![1], "GROUP BY service → column 1"); assert!( measures.contains(&AggIntent::Sum { col: Some(3) }), @@ -325,7 +324,7 @@ async fn count_ranked_topk_is_heavy_hitter() { async fn count_ranked_topk_via_alias_is_also_heavy_hitter() { // Regression for #20: aliasing `COUNT(*)` in the ORDER BY used to defeat the // SQL front-end gate. The positional `canonicalize` pass now promotes it too, - // so the aliased and inline forms produce an identical canonical tree. + // so the aliased and inline forms produce an identical canonical DAG. let inline = lower( "SELECT service, COUNT(*) FROM metrics GROUP BY service ORDER BY COUNT(*) DESC LIMIT 10", ) @@ -523,7 +522,7 @@ async fn join_predicate_disambiguates_shared_column_name() { FROM metrics JOIN hosts ON metrics.service = hosts.service", ) .await; - let join = find_join(&qe).expect("expected a Join in the tree"); + let join = find_join(&qe).expect("expected a Join in the DAG"); assert_eq!( join_eq_columns(join), [1, 4], @@ -544,7 +543,7 @@ async fn derived_table_join_disambiguates_via_alias() { JOIN (SELECT service, region FROM hosts) b ON a.service = b.service", ) .await; - let join = find_join(&qe).expect("expected a Join in the tree"); + let join = find_join(&qe).expect("expected a Join in the DAG"); assert_eq!( join_eq_columns(join), [0, 2], @@ -562,7 +561,7 @@ async fn derived_table_select_star_join_disambiguates_via_alias() { ON a.service = b.service", ) .await; - let join = find_join(&qe).expect("expected a Join in the tree"); + let join = find_join(&qe).expect("expected a Join in the DAG"); let [l, r] = join_eq_columns(join); assert_ne!( l, r, @@ -580,7 +579,7 @@ async fn self_join_disambiguates_via_aliases() { FROM metrics a JOIN metrics b ON a.service = b.service", ) .await; - let join = find_join(&qe).expect("expected a self-Join in the tree"); + let join = find_join(&qe).expect("expected a self-Join in the DAG"); assert_eq!( join_eq_columns(join), [1, 5], @@ -990,7 +989,7 @@ fn all_intents(root: &Rc) -> Vec { async fn derived_table_aggregate_over_aggregate_nests() { // `MAX(s)` over a derived table `(SELECT service, SUM(bytes) AS s … GROUP BY // service)` — the SQL counterpart of PromQL function nesting (issue #27). - // Both reductions survive into the canonical tree: an outer `Max` over + // Both reductions survive into the canonical DAG: an outer `Max` over // the inner `Sum`. let qe = lower( "SELECT MAX(s) FROM \ @@ -1400,8 +1399,8 @@ async fn count_distinct_carries_its_input_column() { #[tokio::test] async fn quantile_and_count_distinct_over_an_expression_bind_the_derived_column() { // A SQL aggregate has no "sample value" to fall back on, so an expression - // argument must never reach the canonical tree as `col: None` (#115). - // Since #110 it reaches the canonical tree as `col: Some(derived)` + // argument must never reach the canonical DAG as `col: None` (#115). + // Since #110 it reaches the canonical DAG as `col: Some(derived)` // instead of being rejected. for q in [ "SELECT approx_percentile_cont(bytes * 8, 0.95) FROM metrics", @@ -1548,7 +1547,7 @@ async fn time_bucketing_keeps_the_scan_predicate() { #[tokio::test] async fn a_plain_group_by_inserts_no_projection() { - // Queries that lowered before #110 must keep their exact tree shape — the + // Queries that lowered before #110 must keep their exact DAG shape — the // projection appears only when something actually needs materializing. for q in [ "SELECT service, SUM(bytes) FROM metrics GROUP BY service", @@ -3075,7 +3074,6 @@ async fn qualified_reducer_argument_over_a_join_reads_its_own_side() { assert_eq!(cols, [side], "{sql}"); } } - /// DataFusion 54 compares `ROW_NUMBER()` (UInt64) with an Int64 literal /// through `Decimal128(20, 0)`; canonical integers are all Int64, so the /// widening cast lowers as a no-op. diff --git a/crates/frontend-sql/tests/temporal_types.rs b/crates/frontend-sql/tests/temporal_types.rs index 88e13b408..7e12c6f33 100644 --- a/crates/frontend-sql/tests/temporal_types.rs +++ b/crates/frontend-sql/tests/temporal_types.rs @@ -1,4 +1,4 @@ -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::{ pre_asap::schema::{DataType, Field, Schema}, types::AccuracyTarget, diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 8eeb41743..35822710d 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -27,21 +27,19 @@ pub mod fixtures { pub fn lower_promql( query: &str, accuracy: AccuracyTarget, - ) -> Result, asap_frontend_promql::unified::PromqlError> { + ) -> Result, asap_frontend_promql::PromqlError> { match lower_promql_root(query, accuracy)? { asap_types::ir::QueryRoot::Operator(node) => Ok(node), - _ => Err( - asap_frontend_promql::unified::PromqlError::UnsupportedFeature( - "expected vector root".into(), - ), - ), + _ => Err(asap_frontend_promql::PromqlError::UnsupportedFeature( + "expected vector root".into(), + )), } } pub fn lower_promql_root( query: &str, accuracy: AccuracyTarget, - ) -> Result { + ) -> Result { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, @@ -66,7 +64,7 @@ pub mod fixtures { ..Default::default() }), }; - let mut lowered = asap_frontend_promql::unified::lower_promql_query_workload(&workload, 0)?; + let mut lowered = asap_frontend_promql::lower_promql_query_workload(&workload, 0)?; Ok(lowered.remove(0)) } diff --git a/crates/integration-tests/tests/frontend_timestamps.rs b/crates/integration-tests/tests/frontend_timestamps.rs index 3b5cec074..300f0cee7 100644 --- a/crates/integration-tests/tests/frontend_timestamps.rs +++ b/crates/integration-tests/tests/frontend_timestamps.rs @@ -1,6 +1,6 @@ //! Cross-frontend evaluation-time semantics (issues #46 and #184). -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::fixtures::lower_promql_root; use asap_types::ir::{NonASAPOp, ScalarExpr}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; diff --git a/crates/integration-tests/tests/kll_pane_execution.rs b/crates/integration-tests/tests/kll_pane_execution.rs index 0b5fec6a6..4869d681b 100644 --- a/crates/integration-tests/tests/kll_pane_execution.rs +++ b/crates/integration-tests/tests/kll_pane_execution.rs @@ -2,10 +2,10 @@ mod physical_common; use asap_physical_operators::{ operators::{Operator, SummaryEvaluation}, + physical_planner::{CompiledPhysicalDAG, InputContract, Source}, plan::{PhysicalDAG, PhysicalOperator, PlanProperties}, runtime::{Input, Limits, OutputStream, RunContext, Scope}, summary_kernels::datasketches_kll::DatasketchesKLLAccumulator, - unified_physical_planner::{CompiledPhysicalDAG, InputContract, Source}, values::{Batch, SchemaRef, Value}, AggregateCore, Error, }; diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index 55c8f7085..e5b2c0a19 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -1,5 +1,5 @@ //! #511 examples: source text → unified dag → summary rewrite → flat export. -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; use asap_types::post_asap::{ ExactKind, ExactParams, FieldDataType, GroupingStrategy, SummaryUpdate, @@ -140,9 +140,9 @@ async fn sql_scalar_subquery_retains_its_cardinality_contract() { #[tokio::test] async fn sql_sum_example_executes_with_sql_null_semantics() { use asap_physical_operators::{ + physical_planner::{compile, InputContract, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{compile, InputContract, Source}, - unified_sources::{DataSources, MemorySource}, + sources::{DataSources, MemorySource}, values::{Batch, Value}, }; use futures::StreamExt; @@ -265,8 +265,8 @@ async fn batch_planning_replaces_and_shares_summary_operators() { SummaryMaintenanceLifecycleCostInputs, }; use asap_physical_operators::{ + physical_planner::{compile, InputContract}, runtime::Scope, - unified_physical_planner::{compile, InputContract}, values::{Batch, Value}, }; use asap_planner::{e2e_plan, FrontendInput, UserInput}; diff --git a/crates/integration-tests/tests/operator_sharing.rs b/crates/integration-tests/tests/operator_sharing.rs index 0e926fe03..0e7f96c35 100644 --- a/crates/integration-tests/tests/operator_sharing.rs +++ b/crates/integration-tests/tests/operator_sharing.rs @@ -10,7 +10,7 @@ use std::rc::Rc; use asap_aware_mapping::{search_workload, DefaultCostModel}; -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::post_asap::post_asap_dag; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 8420331be..2f03f459c 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -1,7 +1,7 @@ use asap_physical_operators::{ operators::Operator, + physical_planner::{CompiledPhysicalDAG, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{CompiledPhysicalDAG, Source}, values::Batch, }; use futures::{executor::block_on, StreamExt}; diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index d1befc4b2..5a0b93e16 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -16,9 +16,9 @@ use asap_integration_tests::fixtures::lower_promql; use asap_physical_operators::{ factory::create_planner_accumulator, operators::Operator, + physical_planner::{precompute, Source}, runtime::{Limits, RunContext, Scope}, summary_kernels::{exact::ExactAccumulator, weighted_frequency::WeightedFrequency}, - unified_physical_planner::{precompute, Source}, values::{Batch, Value}, AggregateCore, KeyByLabelValues, Statistic, }; @@ -141,7 +141,7 @@ fn execute( ); }); let program = serde_json::from_slice::< - asap_physical_operators::unified_physical_planner::CompiledPhysicalDAG, + asap_physical_operators::physical_planner::CompiledPhysicalDAG, >(&serde_json::to_vec(&program).unwrap()) .unwrap(); let schema = precompute::raw_sample_schema(); diff --git a/crates/integration-tests/tests/sql_to_physical.rs b/crates/integration-tests/tests/sql_to_physical.rs index f24ad5a32..fb3bfa1df 100644 --- a/crates/integration-tests/tests/sql_to_physical.rs +++ b/crates/integration-tests/tests/sql_to_physical.rs @@ -1,11 +1,11 @@ //! SQL frontend, candidate selection, physical compilation and fresh-run execution. mod physical_common; use asap_aware_mapping::{search_workload, DefaultCostModel}; -use asap_frontend_sql::unified::{lower_sql, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_physical_operators::{ + physical_planner::{compile, InputContract, Source}, runtime::{Limits, RunContext, Scope}, - unified_physical_planner::{compile, InputContract, Source}, - unified_sources::{DataSources, MemorySource}, + sources::{DataSources, MemorySource}, values::{Batch, Value}, }; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 531a24623..2b0b9e02a 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -24,7 +24,7 @@ use asap_aware_mapping::{ search_workload, ASAPStrategies, DefaultCostModel, Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, }; -use asap_frontend_sql::unified::{lower_sql, lower_sql_dialect, SqlCatalog}; +use asap_frontend_sql::{lower_sql, lower_sql_dialect, SqlCatalog}; use asap_integration_tests::post_asap::post_asap_dag; use asap_types::ir::operator_properties::Reduction; use asap_types::ir::physical_export::{EdgeRole, PhysicalASAPNodeId, PhysicalASAPOperatorPayload}; diff --git a/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs b/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs index 8d24c3d23..b297ed518 100644 --- a/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs +++ b/crates/integration-tests/tests/summary_maintenance_lifecycle_e2e.rs @@ -15,7 +15,7 @@ use asap_aware_mapping::{ SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleCostInputs, SummaryMaintenanceLifecycleRejection, WorkloadDemand, }; -use asap_frontend_promql::unified::lower_promql_workload; +use asap_frontend_promql::lower_promql_workload; use asap_types::ir::OperatorNode; use asap_types::post_asap::{ EvaluationSchedule, SummaryMaintenanceLifecycle, SummaryMaintenanceMode, @@ -273,8 +273,8 @@ mod physical_common; #[test] fn continuous_lifecycle_compiles_and_executes_spatial_kll() { use asap_physical_operators::{ + physical_planner::{compile_candidate, InputContract}, runtime::Scope, - unified_physical_planner::{compile_candidate, InputContract}, values::{Batch, Value}, }; use asap_types::{post_asap::FieldDataType, pre_asap::DataType}; @@ -340,7 +340,7 @@ fn continuous_lifecycle_compiles_and_executes_spatial_kll() { ) .unwrap(); let mut priced = 0; - let feedback = asap_physical_operators::unified_physical_planner::select_candidate( + let feedback = asap_physical_operators::physical_planner::select_candidate( vec![ rejected.map(|candidate| { ( @@ -353,7 +353,7 @@ fn continuous_lifecycle_compiles_and_executes_spatial_kll() { |_| { priced += 1; Ok(Some( - asap_physical_operators::unified_physical_planner::CandidateCost { + asap_physical_operators::physical_planner::CandidateCost { workload_scope: "dashboard".into(), horizon_seconds: 100., total_cost: 1000., @@ -458,10 +458,8 @@ fn lifecycle_timed_dag( let mut lowered = lower_promql_workload(&workload, 0).unwrap().remove(0); if query.contains(" by(") { lowered = - asap_physical_operators::unified_physical_planner::promql_rows::with_series_identity( - &lowered, - ) - .unwrap(); + asap_physical_operators::physical_planner::promql_rows::with_series_identity(&lowered) + .unwrap(); } let root = selected_plan_for_lowered(&workload, lowered, &FullyCostedRuntime, Horizon(100.)).root; @@ -496,8 +494,7 @@ fn lifecycle_timed_dag( /// Compile inputs for a timed DAG: its raw source, available at either phase. fn raw_inputs( dag: &asap_types::ir::physical_export::PhysicalASAPDAG, -) -> std::collections::BTreeMap -{ +) -> std::collections::BTreeMap { let raw = dag .nodes .iter() @@ -512,9 +509,9 @@ fn raw_inputs( .unwrap(); std::collections::BTreeMap::from([( raw.id as u64, - asap_physical_operators::unified_physical_planner::InputContract::bounded( - std::sync::Arc::new(raw.output_schema.clone()), - ), + asap_physical_operators::physical_planner::InputContract::bounded(std::sync::Arc::new( + raw.output_schema.clone(), + )), )]) } @@ -549,8 +546,8 @@ fn planner_lifecycle_selection_reproduces_strategy_timing() { #[test] fn chosen_lifecycle_timing_decides_precompute_contents() { use asap_physical_operators::{ + physical_planner::{compile_candidate, frontier_from_timing}, runtime::Scope, - unified_physical_planner::{compile_candidate, frontier_from_timing}, values::{Batch, Value}, }; use asap_types::{post_asap::FieldDataType, pre_asap::DataType}; @@ -643,7 +640,7 @@ fn chosen_lifecycle_timing_decides_precompute_contents() { /// Ephemeral. Covers the KLL quantile fixture and grouped Rate→Sum. #[test] fn lifecycle_timing_cuts_one_compilation() { - use asap_physical_operators::unified_physical_planner::{ + use asap_physical_operators::physical_planner::{ compile, compile_candidate, cut_candidate, frontier_from_timing, }; for query in ["quantile(0.99, latency)", "sum by(job)(rate(m[1m]))"] { @@ -702,12 +699,12 @@ fn chosen_population_lifecycle_decides_precompute_contents() { maintained_population::MaintainedPopulationStrategy, }; use asap_physical_operators::{ - runtime::Scope, - unified_physical_planner::{ + physical_planner::{ compile_candidate, promql_rows::{series_row, with_series_identity}, InputContract, }, + runtime::Scope, values::{Batch, Value}, }; use asap_types::post_asap::maintained_population::PopulationInput; @@ -770,7 +767,7 @@ fn chosen_population_lifecycle_decides_precompute_contents() { .unwrap(); let (raw_id, schema) = (raw.id as u64, Arc::new(raw.output_schema.clone())); let frontier = - asap_physical_operators::unified_physical_planner::frontier_from_timing(&dag).unwrap(); + asap_physical_operators::physical_planner::frontier_from_timing(&dag).unwrap(); let candidate = compile_candidate( &dag, BTreeMap::from([(raw_id, InputContract::bounded(schema.clone()))]), @@ -847,16 +844,15 @@ fn chosen_population_lifecycle_decides_precompute_contents() { #[test] fn grouped_rate_sum_placement_is_a_lifecycle_choice() { use asap_aware_mapping::enumerate_summary_maintenance_lifecycles; - use asap_physical_operators::unified_physical_planner::{compile_candidate, InputContract}; + use asap_physical_operators::physical_planner::{compile_candidate, InputContract}; use asap_types::post_asap::{ExactKind, FieldDataType}; use std::{collections::BTreeMap, sync::Arc}; let workload = quantile_workload("sum by(job)(rate(m[1m]))"); - let root = - asap_physical_operators::unified_physical_planner::promql_rows::with_series_identity( - &lower_promql_workload(&workload, 0).unwrap().remove(0), - ) - .unwrap(); + let root = asap_physical_operators::physical_planner::promql_rows::with_series_identity( + &lower_promql_workload(&workload, 0).unwrap().remove(0), + ) + .unwrap(); let is_exact = |node: &OperatorNode, kind: ExactKind| { matches!(&node.operator, asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(k, _), .. @@ -927,7 +923,7 @@ fn grouped_rate_sum_placement_is_a_lifecycle_choice() { }) .unwrap(); let frontier = - asap_physical_operators::unified_physical_planner::frontier_from_timing(&dag).unwrap(); + asap_physical_operators::physical_planner::frontier_from_timing(&dag).unwrap(); let [boundary] = frontier.as_slice() else { panic!("one precompute output, got {frontier:?}"); }; @@ -1004,10 +1000,8 @@ fn execute_timed( samples: &[(&str, &str, i64, f64)], ) -> Vec { use asap_physical_operators::{ + physical_planner::{compile_candidate, frontier_from_timing, promql_rows, InputContract}, runtime::Scope, - unified_physical_planner::{ - compile_candidate, frontier_from_timing, promql_rows, InputContract, - }, values::{Batch, Value}, }; use asap_types::{ir::physical_export::PhysicalASAPOperatorPayload, pre_asap::Source}; @@ -1071,12 +1065,11 @@ fn execute_timed( &frontier, ) .unwrap(); - let raw_sources = - |plan: &asap_physical_operators::unified_physical_planner::CompiledPhysicalDAG| { - plan.input_contracts() - .filter_map(|(id, _)| raw.get(&id).map(|(schema, name)| (id, batch(schema, name)))) - .collect::>() - }; + let raw_sources = |plan: &asap_physical_operators::physical_planner::CompiledPhysicalDAG| { + plan.input_contracts() + .filter_map(|(id, _)| raw.get(&id).map(|(schema, name)| (id, batch(schema, name)))) + .collect::>() + }; let mut query_sources = raw_sources(&candidate.query); if let Some(precompute) = &candidate.precompute { let stored = physical_common::execute( diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index 77de25378..63ed6c645 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -14,12 +14,12 @@ use asap_types::parsed_workload::{ParsedWorkload, ParsedWorkloadError}; use asap_types::workload::{PlanningWorkload, QueryLanguage, SqlDialect, WorkloadError}; -use asap_frontend_metricsql::unified::{lower_metricsql_query_workload, MetricsqlError}; -use asap_frontend_promql::unified::{ +use asap_frontend_metricsql::{lower_metricsql_query_workload, MetricsqlError}; +use asap_frontend_promql::{ lower_promql_query_workload, lower_promql_query_workload_with_histograms, HistogramCatalog, PromqlError, }; -use asap_frontend_sql::unified::{lower_sql_dialect, SqlCatalog, SqlError}; +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError}; // The optimization stage's vocabulary is this facade's vocabulary too: a caller // configures the same models and reads the same output whether it goes through diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index 5231c341d..a1cf15d05 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -10,7 +10,7 @@ use asap_aware_mapping::replacement::default_strategies_with_evidence; use asap_aware_mapping::{ search_workload_with_targets, Horizon, LifecycleInput, SummaryMaintenanceLifecycleCapabilities, }; -use asap_frontend_sql::unified::{lower_sql_dialect, SqlCatalog}; +use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_planner::{e2e_plan, FrontendInput, PlanError, UserInput, UserInputError}; use asap_types::pre_asap::schema::{DataType, Field, Schema}; use asap_types::types::AccuracyTarget; diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index 2ce32f0d7..73a2c62ff 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -19,8 +19,8 @@ use asap_aware_mapping::{ CostModel, CostRate, DefaultCostModel, Horizon, LifecycleInput, SummaryMaintenanceCapabilities, SummaryMaintenanceLifecycleCapabilities, SummaryMaintenanceLifecycleCostInputs, }; -use asap_frontend_promql::unified::lower_promql_workload; -use asap_frontend_sql::unified::SqlCatalog; +use asap_frontend_promql::lower_promql_workload; +use asap_frontend_sql::SqlCatalog; use asap_planner::{e2e_plan, FrontendInput, UserInput}; use asap_types::post_asap::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, ProbabilityExpr, ResultGuarantee, diff --git a/crates/sql-function-catalog/src/lib.rs b/crates/sql-function-catalog/src/lib.rs index be79e7a71..f2aedbaa9 100644 --- a/crates/sql-function-catalog/src/lib.rs +++ b/crates/sql-function-catalog/src/lib.rs @@ -323,7 +323,7 @@ pub const CLICKHOUSE_BUILTINS: &[ClickHouseBuiltin] = &[ /// /// This list exists for one reason: `asap-frontend-sql`'s DataFusion-registry /// drift test (issue #225, item 3 -- see -/// `asap_frontend_sql::unified::sql::catalog_drift`) walks a real `SessionContext`'s +/// `asap_frontend_sql::sql::catalog_drift`) walks a real `SessionContext`'s /// resolved aggregate names and requires every one to be either in /// [`NATIVE_FUNCTIONS`], a [`CLICKHOUSE_BUILTINS`] name, or listed here. A /// name landing here is a *recorded decision*, not a silenced test failure -- From b26bfdc5a9d42d75d513d12f5cba08f2f5fd5683 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:53:09 +0000 Subject: [PATCH 3/3] fix: pass the ingestion interval in the MetricsQL consumer check; review cleanups The external-consumer script calls lower_metricsql with the ingestion interval it now takes, and removes its temp crate on exit. Also: the physical-operators README names the input a Planner PhysicalASAPDAG, expressions/mod.rs names CompiledExpression directly, lowering test locals and prose say dag/DAG, and the default_rollup-without-range test is dropped since rollups_without_an_explicit_range_are_rejected covers it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W7qG9aFyPij5uWsyAJCxDW --- crates/asap-physical-operators/README.md | 2 +- .../src/expressions/mod.rs | 4 ++-- crates/frontend-metricsql/tests/lowering.rs | 17 ++++------------- crates/frontend-promql/tests/promql_lowering.rs | 4 ++-- tools/verify_metricsql_external_consumer.sh | 2 ++ 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/crates/asap-physical-operators/README.md b/crates/asap-physical-operators/README.md index 9c39b8c33..35b5c1dde 100644 --- a/crates/asap-physical-operators/README.md +++ b/crates/asap-physical-operators/README.md @@ -47,7 +47,7 @@ assert!(matches!(batch.rows()[0][0], Value::Int64(-7))); # Ok::<(), asap_physical_operators::dag::Error>(()) ``` -`physical_planner::compile` accepts a logical Post-ASAP DAG (`PhysicalASAPDAG`) and typed input contracts. +`physical_planner::compile` accepts a Planner `PhysicalASAPDAG` and typed input contracts. The resulting candidate is instantiated with deployment readers after selection. It rejects unsupported operations and schema mismatches before starting a source. Implement `PhysicalOperator` for a deployment source, including asynchronous I/O; computation operators remain in diff --git a/crates/asap-physical-operators/src/expressions/mod.rs b/crates/asap-physical-operators/src/expressions/mod.rs index aad035516..cb7c00f8a 100644 --- a/crates/asap-physical-operators/src/expressions/mod.rs +++ b/crates/asap-physical-operators/src/expressions/mod.rs @@ -15,7 +15,7 @@ pub enum Expression { left: Box, right: Box, }, - Planner(Box), + Planner(Box), Column(usize), ExactFloat64(usize), FiniteFloat64(Box), @@ -54,7 +54,7 @@ pub enum Expression { IsNull(Box), } impl Expression { - pub fn planner(expression: crate::expressions::CompiledExpression) -> Self { + pub fn planner(expression: CompiledExpression) -> Self { Self::Planner(Box::new(expression)) } pub(crate) fn dtype(&self, input: &SchemaRef) -> Result<(DataType, bool), Error> { diff --git a/crates/frontend-metricsql/tests/lowering.rs b/crates/frontend-metricsql/tests/lowering.rs index 84dd42cea..549edfadd 100644 --- a/crates/frontend-metricsql/tests/lowering.rs +++ b/crates/frontend-metricsql/tests/lowering.rs @@ -18,13 +18,13 @@ fn lower(query: &str) -> Rc { #[test] fn selector_range_aggregate_and_call_share_the_canonical_shape() { let query = r#"sum by (job) (rate(http_requests_total{status=~"5.."}[5m]))"#; - let tree = lower(query); + let dag = lower(query); let NonASAPOp::Aggregate { reduction, measures, child, .. - } = tree.expect_non_asap() + } = dag.expect_non_asap() else { panic!("expected outer aggregate"); }; @@ -48,13 +48,13 @@ fn selector_range_aggregate_and_call_share_the_canonical_shape() { #[test] fn default_rollup_with_explicit_range_is_last_over_time() { - let tree = lower("default_rollup(cpu_usage[5m])"); + let dag = lower("default_rollup(cpu_usage[5m])"); let NonASAPOp::Aggregate { reduction, measures, child, .. - } = tree.expect_non_asap() + } = dag.expect_non_asap() else { panic!("expected aggregate"); }; @@ -66,15 +66,6 @@ fn default_rollup_with_explicit_range_is_last_over_time() { ); } -#[test] -fn implicit_default_rollup_requires_runtime_step() { - let error = - lower_metricsql("default_rollup(cpu_usage)", AccuracyTarget::Exact, INTERVAL).unwrap_err(); - assert!( - matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("evaluation step")) - ); -} - #[test] fn keep_metric_names_is_preserved_in_ast_and_rejected_without_lineage() { let ast = parse_metricsql("rate(requests_total[5m]) keep_metric_names").unwrap(); diff --git a/crates/frontend-promql/tests/promql_lowering.rs b/crates/frontend-promql/tests/promql_lowering.rs index 1aee42cdc..6ccddeec5 100644 --- a/crates/frontend-promql/tests/promql_lowering.rs +++ b/crates/frontend-promql/tests/promql_lowering.rs @@ -900,7 +900,7 @@ fn binary_op_binds_each_branch_against_its_own_schema() { ); } -/// Collect every `AggIntent` in the dag, root-to-leaf. +/// Collect every `AggIntent` in the DAG, root-to-leaf. fn all_intents(e: &OperatorNode) -> Vec { let mut out = Vec::new(); collect_intents(e, &mut out); @@ -1501,7 +1501,7 @@ fn explicit_range_selector_is_a_range_time_range() { #[test] fn instant_and_range_selectors_of_equal_length_stay_distinct() { - // The kind is part of the shape: a 1s range selector is not the same dag as + // The kind is part of the shape: a 1s range selector is not the same DAG as // the 1s instant lookback injected around a bare selector. assert_ne!(lower("up"), lower("up[1s]")); } diff --git a/tools/verify_metricsql_external_consumer.sh b/tools/verify_metricsql_external_consumer.sh index 7723357fe..9da6e35f2 100644 --- a/tools/verify_metricsql_external_consumer.sh +++ b/tools/verify_metricsql_external_consumer.sh @@ -3,6 +3,7 @@ set -euo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) consumer_dir=$(mktemp -d "${TMPDIR:-/tmp}/asap-metricsql-consumer.XXXXXX") +trap 'rm -rf "$consumer_dir"' EXIT mkdir -p "$consumer_dir/src" cat > "$consumer_dir/Cargo.toml" <