diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 510d3aa9b..63b866090 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -12,7 +12,6 @@ name = "control_plane" path = "src/main.rs" [dependencies] -asap-physical-operators.workspace = true tokio = { version = "1", features = ["full"] } axum = { version = "0.7", features = ["ws"] } futures-util = "0.3" @@ -33,6 +32,7 @@ parking_lot = "0.12" prometheus = { version = "0.13", default-features = false, features = ["process"] } tonic = { version = "0.12", features = ["gzip"] } asap_types.workspace = true +asap-physical-operators.workspace = true # Planner's IR, replacement search, and frontends share the workspace pin. planner-types.workspace = true diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index 595f97321..dc356a00c 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -1,22 +1,11 @@ -//! HTTP client that pushes a freshly-generated `StreamingConfig` YAML -//! to the ASAPQuery-backend's `POST /api/v1/streaming-config` endpoint. -//! -//! This is the control-plane-side **producer** of the PR E phase 1 / phase 2 -//! hot-reload contract that landed in ASAPQuery-backend PRs #10 and #12. -//! The replanner calls into this module immediately after generating a -//! new plan so the backend's active `StreamingConfig` is updated without -//! a restart and subsequent queries observe the new aggregation layout. -//! -//! The client is **fire-and-forget at the call site** — the replanner -//! awaits the POST but doesn't block its own return on the outcome. -//! Errors are logged at WARN; the control plane is expected to be tolerant -//! of transient backend unavailability because the next replan cycle -//! will try again with the latest plan. +//! Publish and activate complete physical-plan generations on the backend. use std::time::Duration; +#[cfg(test)] use anyhow::{Context, Result}; use reqwest::Client; +#[cfg(test)] use tracing::debug; #[cfg(test)] use tracing::warn; @@ -92,7 +81,7 @@ fn classify_http_status(status: reqwest::StatusCode, body: String, what: &str) - } } -/// Minimal HTTP client for ASAPQuery-backend's streaming-config endpoint. +/// Minimal HTTP client for ASAPQuery-backend's physical-plan endpoint. /// Built once at control-plane startup from the /// `CONTROL_PLANE_BACKEND_ENDPOINT` environment variable and shared /// via `Arc` with the replanner. @@ -105,7 +94,7 @@ pub struct BackendClient { impl BackendClient { /// Construct a client pointing at the backend's plan-push endpoint. /// `endpoint` should be the full URL, e.g. - /// `http://backend.svc:8088/api/v1/streaming-config`. + /// `http://backend.svc:8088/api/v1/physical-plan`. /// /// A 5-second timeout bounds the duration a slow or unreachable /// backend can stall the replanner — consistent with the symmetric @@ -136,11 +125,12 @@ impl BackendClient { &self.endpoint } - /// POST the given `StreamingConfig` YAML to the backend. Returns + /// Test-only transport helper: POST the given YAML to the backend. Returns /// `Ok(())` on any 2xx status, otherwise an error carrying the /// status code and response body. The caller (typically /// [`Replanner::replan_metric`]) logs the error and moves on — the /// next replan cycle will retry with the latest plan. + #[cfg(test)] pub async fn push_streaming_config(&self, yaml: String) -> Result<()> { debug!( endpoint = %self.endpoint, @@ -212,6 +202,7 @@ impl BackendClient { /// that don't need retry semantics keep their `anyhow::Result` /// shape. The retry layer in `emit::backend_push` uses this typed /// variant. + #[cfg(test)] pub async fn post_streaming_config_json_typed( &self, json: String, @@ -245,7 +236,7 @@ impl BackendClient { #[cfg(test)] /// Post backend storage-routing JSON. Derive the URL by replacing the - /// `/api/v1/streaming-config` suffix with `/api/v1/storage_routing`; URLs + /// `/api/v1/physical-plan` suffix with `/api/v1/storage_routing`; URLs /// without that suffix are used verbatim. pub async fn post_storage_routing_json(&self, json: String) -> Result<()> { let url = derive_storage_routing_url(&self.endpoint); @@ -364,25 +355,18 @@ impl BackendClient { } fn derive_physical_plan_url(endpoint: &str) -> String { - const DASH: &str = "/api/v1/streaming-config"; - const UNDERSCORE: &str = "/api/v1/streaming_config"; - const PHYSICAL: &str = "/api/v1/physical-plan"; - endpoint - .strip_suffix(DASH) - .or_else(|| endpoint.strip_suffix(UNDERSCORE)) - .map(|base| format!("{base}{PHYSICAL}")) - .unwrap_or_else(|| endpoint.to_string()) + endpoint.to_string() } -/// Map a streaming-config endpoint URL to the sibling storage-routing +/// Map a physical-plan endpoint URL to the sibling storage-routing /// endpoint by rewriting the trailing path component. URLs that don't -/// end with `/api/v1/streaming-config` (or `/api/v1/streaming_config` — +/// end with `/api/v1/physical-plan` (or `/api/v1/physical_plan` — /// either spelling is supported) pass through unchanged so tests can /// inject a mock-server URL directly. #[cfg(test)] fn derive_storage_routing_url(endpoint: &str) -> String { - const STREAMING_PATH_DASH: &str = "/api/v1/streaming-config"; - const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/streaming_config"; + const STREAMING_PATH_DASH: &str = "/api/v1/physical-plan"; + const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/physical_plan"; const ROUTING_PATH: &str = "/api/v1/storage_routing"; if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_DASH) { return format!("{stripped}{ROUTING_PATH}"); @@ -429,7 +413,7 @@ mod tests { async fn start_mock_backend(sink: SharedSink, status: axum::http::StatusCode) -> String { let app = Router::new() .route( - "/api/v1/streaming-config", + "/api/v1/physical-plan", post( move |State(sink): State, body: axum::body::Bytes| async move { let yaml = String::from_utf8_lossy(&body).to_string(); @@ -445,7 +429,7 @@ mod tests { axum::serve(listener, app).await.unwrap(); }); tokio::time::sleep(Duration::from_millis(50)).await; - format!("http://{addr}/api/v1/streaming-config") + format!("http://{addr}/api/v1/physical-plan") } #[tokio::test] @@ -481,7 +465,7 @@ mod tests { #[tokio::test] async fn push_or_log_swallows_errors() { // Point at an unreachable port so the request fails fast. - let client = BackendClient::new("http://127.0.0.1:1/api/v1/streaming-config"); + let client = BackendClient::new("http://127.0.0.1:1/api/v1/physical-plan"); // Must not panic or propagate — fire-and-forget semantics. push_or_log(&client, "cpu_usage", "content".to_string()).await; } @@ -522,15 +506,15 @@ mod tests { /// storage-routing-URL derivation rewrites the path /// component when the configured endpoint ends in - /// `/api/v1/streaming-config`, leaving everything else untouched. + /// `/api/v1/physical-plan`, leaving everything else untouched. #[test] fn storage_routing_url_rewrites_streaming_path() { assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/streaming-config"), + derive_storage_routing_url("http://backend:8088/api/v1/physical-plan"), "http://backend:8088/api/v1/storage_routing" ); assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/streaming_config"), + derive_storage_routing_url("http://backend:8088/api/v1/physical_plan"), "http://backend:8088/api/v1/storage_routing" ); } @@ -578,7 +562,7 @@ mod tests { let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); - let client = BackendClient::new(format!("http://{addr}/api/v1/streaming-config")); + let client = BackendClient::new(format!("http://{addr}/api/v1/physical-plan")); client .post_catalog_plan_typed(&publication, None, &[]) .await @@ -630,7 +614,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; // Return the streaming-config URL — the client will rewrite // the path before issuing the POST. - format!("http://{addr}/api/v1/streaming-config") + format!("http://{addr}/api/v1/physical-plan") } #[tokio::test] @@ -702,7 +686,7 @@ mod tests { #[tokio::test] async fn typed_connection_refused_is_transient() { // Port 1 on loopback is reserved and refuses connections. - let client = BackendClient::new("http://127.0.0.1:1/api/v1/streaming-config"); + let client = BackendClient::new("http://127.0.0.1:1/api/v1/physical-plan"); let err = client .post_streaming_config_json_typed("{}".to_string()) .await diff --git a/control_plane/src/emit/monitor.rs b/control_plane/src/emit/monitor.rs index 4bd65e6e9..59776f3a8 100644 --- a/control_plane/src/emit/monitor.rs +++ b/control_plane/src/emit/monitor.rs @@ -9,7 +9,7 @@ //! by `asapedgeprocessor.ThresholdConfig`). The edge derives the agg_id //! from the metric name itself, so this block carries no agg_id. //! 2. [`streaming_config_monitor_entry`] — the `monitors[]` JSON object for the -//! backend `StreamingConfig` (consumed by `asap_types::MonitorSpec`), where +//! backend `InstalledPrecomputePlan` (consumed by `asap_types::MonitorSpec`), where //! `agg_id` IS carried and MUST equal [`agg_id_for_metric`]. //! //! The functions are pure so they can be unit-tested and called from whichever @@ -88,7 +88,7 @@ pub fn edge_threshold_block(intent: &MonitorIntent) -> Value { Value::Mapping(m) } -/// Render the backend `StreamingConfig.monitors[]` JSON entry for this intent, +/// Render the backend `InstalledPrecomputePlan.monitors[]` JSON entry for this intent, /// stamping the cross-language `agg_id`. pub fn streaming_config_monitor_entry(intent: &MonitorIntent) -> serde_json::Value { serde_json::json!({ diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index edd4afe08..fae7acbb2 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -41,7 +41,7 @@ struct AppState { /// input identity; incoming telemetry cannot supply its own descriptors. active_summary_catalog: Arc>>>, - /// Shared client for posting streaming configs from HTTP planning and replanning. + /// Shared client for publishing physical plans from HTTP planning and replanning. /// `None` when `CONTROLLER_BACKEND_ENDPOINT` is unset; pushes are then skipped. backend_client: Option>, } @@ -69,13 +69,13 @@ async fn main() { backend_endpoint.as_ref().map(|endpoint| { info!( endpoint = %endpoint, - "ASAPQuery-backend StreamingConfig push enabled" + "ASAPQuery-backend InstalledPrecomputePlan push enabled" ); Arc::new(backend_client::BackendClient::new(endpoint.clone())) }); if backend_client_shared.is_none() { info!( - "ASAPQuery-backend StreamingConfig push disabled \ + "ASAPQuery-backend physical-plan publication disabled \ (set CONTROLLER_BACKEND_ENDPOINT= to enable)" ); } @@ -1000,9 +1000,9 @@ mod api_tests { #[test] fn app_state_backend_client_some_when_constructed_with_url() { let (state, _router) = - test_app_with_backend(Some("http://127.0.0.1:1/api/v1/streaming-config".into())); + test_app_with_backend(Some("http://127.0.0.1:1/api/v1/physical-plan".into())); let bc = state.backend_client.expect("backend_client must be Some"); - assert_eq!(bc.endpoint(), "http://127.0.0.1:1/api/v1/streaming-config"); + assert_eq!(bc.endpoint(), "http://127.0.0.1:1/api/v1/physical-plan"); } // ── POST /api/v1/plan ───────────────────────────────────────────────────── diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 5e12548d8..c53d6fd52 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -6993,8 +6993,8 @@ pub(crate) mod tests { &query_plan, ) .unwrap(); - // V1 has one stored output per definition; arbitrary output IDs are - // rejected before writer/reader agreement is considered. + // Output identity is independent of definition identity, but changing + // only the writer must still invalidate every unchanged reader binding. let mut rebound_writer = bundle.precompute_plan.clone(); rebound_writer.schemas[0] .stored_output_reference @@ -7002,6 +7002,13 @@ pub(crate) mod tests { assert!(rebound_writer .validate_against_catalog(&bundle.summary_catalog) .is_err()); + assert!( + asap_types::plan_publication::validate_stored_output_references( + &rebound_writer, + &query_plan, + ) + .is_err() + ); } #[test] diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs index 378967d87..93056ad3a 100644 --- a/control_plane/src/physical/executable_binding.rs +++ b/control_plane/src/physical/executable_binding.rs @@ -2,6 +2,21 @@ pub use asap_types::executable_plan::*; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum OperatorExecution { + Ingestion, + Query, +} + +/// Every physical operator uses its node's placement; payload kind does not +/// restrict execution phase. Runtime capability is checked separately. +fn operator_execution(node: &planner_types::post_asap::ExecutableDagNode) -> OperatorExecution { + match node.output_state.timing { + planner_types::post_asap::ExecutionTiming::IngestionTime => OperatorExecution::Ingestion, + planner_types::post_asap::ExecutionTiming::QueryTime => OperatorExecution::Query, + } +} + /// Assign backend phases to a selected semantic DAG without changing its nodes. pub fn install_selected_dag( query_id: String, @@ -15,12 +30,21 @@ pub fn install_selected_dag( let mut nodes = std::collections::BTreeMap::new(); let mut precompute_sinks = Vec::new(); for node in &dag.nodes { + if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + .. + } = &node.payload + { + asap_physical_operators::capability::validate_summary_kernel(family, input, grouping) + .map_err(|reason| format!("post-ASAP node {:?}: {reason}", node.id))?; + } + let execution = operator_execution(node); let binding = if let Some(stored_output) = materialization(node.id) { precompute_sinks.push(node.id); BackendNodeBinding::Materialization { stored_output } - } else if node.output_state.timing - == planner_types::post_asap::ExecutionTiming::IngestionTime - { + } else if execution == OperatorExecution::Ingestion { BackendNodeBinding::MaintenanceInput } else { query_node(node.id).map_or(BackendNodeBinding::QueryInput, |query_node| { diff --git a/control_plane/src/physical/post_asap/deployment_expr.rs b/control_plane/src/physical/post_asap/deployment_expr.rs index 4012277eb..b0372b6a6 100644 --- a/control_plane/src/physical/post_asap/deployment_expr.rs +++ b/control_plane/src/physical/post_asap/deployment_expr.rs @@ -59,7 +59,7 @@ pub enum PhysicalExpr { /// Phase ε.1 Mode 2: no sketch processor at the edge — raw OTLP /// forwards to the backend, which builds the sketch at ingest. The /// `family` and `params` are the sketch the backend will build, so - /// the backend's `StreamingConfig` `aggregation_input` is `raw` for + /// the backend's `InstalledPrecomputePlan` `aggregation_input` is `raw` for /// this metric. RawAtEdgeSketchAtBackend { /// Sketch family the backend will build at ingest. diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 21bc52fff..917df4626 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -605,7 +605,7 @@ fn pipeline_l1_to_l4(query: &str, accuracy: AccuracyTarget) -> PhysicalExpr { } /// `quantile_over_time.yaml` — the asap-planner-rs `quantile_over_time` -/// fixture maps to a KLL or DDSketch StreamingConfig row. The control plane +/// fixture maps to a KLL or DDSketch InstalledPrecomputePlan row. The control plane /// path: L1 PromQL parse → L3 `Aggregate{Quantile{0.99}}` over `Window` → /// L4 bind picks Kll (default) or DDSketch. Either is functionally /// equivalent — both are quantile sketches. diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index d6b74cd45..395ce65ca 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -501,7 +501,7 @@ impl WorkloadRegistry { /// Inject (or replace, keyed by `metric_name`) a runtime workload entry. /// Used by the autonomous-allocation apply path to register a synthesized /// monitor so the next replan/repost emits it into the backend - /// `StreamingConfig` (the coordinator then derives the ε-floor `p`). Shared + /// `InstalledPrecomputePlan` (the coordinator then derives the ε-floor `p`). Shared /// across registry clones via the `Arc>` overlay. pub fn insert_runtime(&self, entry: WorkloadEntry) { let mut rt = self diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index a170f3cd7..24d804d2f 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -320,7 +320,7 @@ impl PrecomputeMaterialization { } /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the - /// policy-fingerprint-keyed call sites (e.g. `StreamingConfig`'s + /// policy-fingerprint-keyed call sites (e.g. `InstalledPrecomputePlan`'s /// `HashMap` keys). **Always** equal to /// `self.policy_fingerprint().as_u64()`. The value is content- /// addressed identity, NOT a controller-allocated counter id. diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index c475584a6..1c89096c6 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -369,7 +369,7 @@ mod tests { fn send_sync() {} send_sync::(); let wire = serde_json::json!({ - "schema_version": 1, "query_id": "q", "nodes": [], "edges": [], "root": 0 + "schema_version": 2, "query_id": "q", "nodes": [], "edges": [], "root": 0 }); let document: OwnedPostAsapDag = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(serde_json::to_value(document).unwrap(), wire); diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs index a22352be6..6aad5ad06 100644 --- a/crates/asap_types/src/monitor_spec.rs +++ b/crates/asap_types/src/monitor_spec.rs @@ -38,9 +38,9 @@ impl MonitorFunctional { /// (empty for Sum / whole-stream). See /// `ASAPCollector/docs/continuous-monitoring-tumbling-cost-analysis.md`. /// -/// Stays here (unlike `data_plane::storage_engines::types::StreamingConfig`, +/// Stays here (unlike `data_plane::storage_engines::types::InstalledPrecomputePlan`, /// which holds a `Vec` field) because `control_plane` genuinely -/// needs it: `emit/monitor.rs` builds the `StreamingConfig.monitors[]` JSON +/// needs it: `emit/monitor.rs` builds the `InstalledPrecomputePlan.monitors[]` JSON /// entry by hand and has a regression test asserting that JSON deserializes /// into this exact type. `control_plane` cannot depend on `data_plane` (the /// dependency runs the other way), so this type has to live somewhere both diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index dde5e5cce..16f007bb8 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -3,8 +3,8 @@ //! Derived view over a collection of `PrecomputeMaterialization`s that maps //! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the //! merged-sid-identity-chain replacement for the controller-allocated -//! `aggregation_id`-keyed `HashMap` that `data_plane`'s `StreamingConfig` -//! carries (see `data_plane::storage_engines::types::streaming_config`'s +//! `aggregation_id`-keyed `HashMap` that `data_plane`'s `InstalledPrecomputePlan` +//! carries (see `data_plane::storage_engines::types::installed_precompute_plan`'s //! module doc for why that type lives there, not here). //! //! ## Dual-keyed transition diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 40a90fe36..ec20b7ef2 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -1,6 +1,6 @@ //! `RoutingIndex` — a metric-bucketed structural index over a //! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `StreamingConfig`'s `PrecomputeMaterialization`s, so it represents planned policy +//! `InstalledPrecomputePlan`'s `PrecomputeMaterialization`s, so it represents planned policy //! rather than a reconstruction from ingest side effects. //! //! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] @@ -23,11 +23,11 @@ //! `streaming_snap.policy_registry()` call) still get the Tier-2 win for //! any query with more than one candidate sharing the same snapshot //! (composed PromQL shapes routinely do). Building it once per -//! `StreamingConfig` hot-reload swap instead of once per query — the same +//! `InstalledPrecomputePlan` hot-reload swap instead of once per query — the same //! "cheap, but call at swap time not per query, if it shows up in -//! profiles" note `StreamingConfig::policy_registry`'s own doc comment +//! profiles" note `InstalledPrecomputePlan::policy_registry`'s own doc comment //! already flags — is a further, larger change (it means threading a -//! cached derived value through `StreamingConfigHandle`'s swap path) +//! cached derived value through `InstalledPrecomputePlanHandle`'s swap path) //! and is not done by this type on its own. use std::collections::{BTreeSet, HashMap}; diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 9433816d1..2e665f9c7 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -75,7 +75,7 @@ impl SummaryDefinitionId { } /// The installed writer/reader binding joins deployment identity and semantics. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct StoredOutputReference { pub stored_output_id: StoredOutputId, @@ -91,10 +91,41 @@ impl StoredOutputReference { } } pub fn validate(&self) -> Result<(), SdsError> { + if self.stored_output_id.0 == 0 { + return Err(SdsError("stored output identity must be set".into())); + } self.definition_id.validate() } } +/// Canonical address of one stored DAG output for one population and window. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoredSummaryKey { + pub plan_id: u64, + pub plan_version: u64, + pub stored_output_id: StoredOutputId, + pub population: BTreeMap, + pub window: HalfOpenTimeRange, +} + +impl StoredSummaryKey { + pub fn validate(&self) -> Result<(), SdsError> { + if self.stored_output_id.0 == 0 { + return Err(SdsError("stored output identity must be set".into())); + } + if self.window.start_ms >= self.window.end_ms { + return Err(SdsError("stored summary requires a nonempty window".into())); + } + Ok(()) + } + + pub fn storage_key(&self) -> Result { + self.validate()?; + serde_json::to_string(self).map_err(|e| SdsError(e.to_string())) + } +} + descriptor_id!(SummaryDescriptorId); descriptor_id!(DataDescriptorId); @@ -1349,6 +1380,49 @@ mod tests { inventory.validate().unwrap(); } + // Every component of the persisted output address participates in identity. + #[test] + fn stored_summary_key_binds_plan_output_population_and_window() { + let key = StoredSummaryKey { + plan_id: 1, + plan_version: 2, + stored_output_id: StoredOutputId(101), + population: BTreeMap::from([("service".into(), "api".into())]), + window: HalfOpenTimeRange { + start_ms: 0, + end_ms: 1000, + }, + }; + let mut variants = vec![key.clone(); 5]; + variants[0].plan_id += 1; + variants[1].plan_version += 1; + variants[2].stored_output_id.0 += 1; + variants[3].population.insert("service".into(), "db".into()); + variants[4].window.end_ms += 1; + let canonical = key.storage_key().unwrap(); + for other in variants { + assert_ne!(canonical, other.storage_key().unwrap()); + } + assert_eq!( + serde_json::from_str::(&canonical).unwrap(), + key + ); + } + + // A DAG output has its own identity even when its definition is shared. + #[test] + fn stored_output_identity_is_independent_of_definition() { + let definition_id = SummaryDefinitionId::from_semantics(b"fixture"); + for stored_output_id in [StoredOutputId(101), StoredOutputId(102)] { + StoredOutputReference { + stored_output_id, + definition_id: definition_id.clone(), + } + .validate() + .unwrap(); + } + } + #[test] fn stored_output_and_payload_version_must_match_instance_definition() { let mut instance = observed_instance(InstanceLifecycle::Persistent); diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 23aedab50..de4a448cf 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -112,7 +112,7 @@ fn sketch_meta( config: SketchConfig, ) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), capability: Some(match algorithm { @@ -134,7 +134,7 @@ fn sketch_meta( fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: None, @@ -390,12 +390,14 @@ fn bench_query_precomputes_by_agg(c: &mut Criterion) { g.finish(); } -/// Build a `StreamingConfig` whose single agg-config's content +/// Build a `InstalledPrecomputePlan` whose single agg-config's content /// signature matches every sid registered by `build_precompute_store` /// (metric / `Sum` / no grouping / empty params+filter). With this /// config the reconciler retires nothing — the steady-state ingest /// case, where the per-batch reconcile is pure scan overhead. -fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types::StreamingConfig { +fn matching_streaming_config( + metric: &str, +) -> data_plane::storage_engines::types::InstalledPrecomputePlan { use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; @@ -419,9 +421,14 @@ fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types None, None, ); - let mut map = HashMap::new(); - map.insert(1u64, cfg); - data_plane::storage_engines::types::StreamingConfig::new(map) + let runtime = data_plane::storage_engines::types::InstalledPrecomputePlan::default(); + let plan = control_plane::physical::compiler::PrecomputePlan::build( + runtime.plan().envelope.clone(), + vec![cfg], + &[], + ) + .unwrap(); + data_plane::storage_engines::types::InstalledPrecomputePlan::from_precompute_plan(plan).unwrap() } /// `reconcile_from_streaming_config` ran on EVERY ingest batch and, in diff --git a/data_plane/examples/sketch_db_diag.rs b/data_plane/examples/sketch_db_diag.rs index 90555f5f6..739842f49 100644 --- a/data_plane/examples/sketch_db_diag.rs +++ b/data_plane/examples/sketch_db_diag.rs @@ -52,7 +52,7 @@ fn dd_meta(sid: u64) -> SummarySeriesMetadata { relative_accuracy: 0.01, }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 2b23439a7..7185b4792 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -5,7 +5,7 @@ //! engine via [`OtlpReceiver::with_ingest_state`] — routes both raw metric //! points and pre-built sketches through the precompute engine's worker //! pool. The precompute engine then performs window-aligned aggregation -//! per `StreamingConfig` and writes results to `SketchStore`. +//! per `InstalledPrecomputePlan` and writes results to `SketchStore`. //! //! Architectural flow: //! ```text @@ -621,27 +621,11 @@ fn resolve_bucket_sid_for_agg_config( config.population_key_encoding, &grouping_pairs, )?; - let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); - let sid = ingest_state.series_resolver.resolve_with_reactivation( - &config.metric, + let sid = ingest_state.summary_store.resolve_output_storage_handle( + &ingest_state.series_resolver, + config.policy_fingerprint().into(), &fp, - &agg_kind_canonical, - |sid| { - ingest_state - .summary_store - .validate_routed_catalog_generation(captured_generation)?; - let activation = ingest_state - .summary_store - .authorize_series_reactivation(sid, config.policy_fingerprint().into())?; - if activation - .as_deref() - .is_some_and(|generation| Some(generation) != captured_generation) - { - return Err("stale OTLP generation cannot reactivate series".into()); - } - Ok(activation) - }, + captured_generation, )?; let policy_fp = asap_types::PolicyFingerprint(config.policy_fp_u64()); Ok((sid, policy_fp)) @@ -663,7 +647,7 @@ async fn route_otlp_to_precompute( .map(Arc::new); let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.streaming_config.clone()) + .map(|plan| plan.installed_precompute_plan.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); let agg_configs = snap.materializations(); // Reconcile sid lifecycle using the current streaming-config snapshot. @@ -914,7 +898,7 @@ async fn route_modified_otlp_sketches_to_precompute( let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.streaming_config.clone()) + .map(|plan| plan.installed_precompute_plan.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); let catalog_generation = active_physical_plan_snapshot .as_ref() @@ -1225,7 +1209,7 @@ async fn route_modified_otlp_sketches_to_precompute( // because `fp=""` is a perfectly good mint/lookup key. let resolved_sid: Option = if dp.series_id != 0 && attrs_pairs.is_empty() { let sid = dp.series_id; - if ingest_state.summary_store.instance(sid).is_some() { + if ingest_state.summary_store.is_current_storage_handle(sid) { Some(sid) } else { unknown_sids.push(sid); @@ -1253,54 +1237,36 @@ async fn route_modified_otlp_sketches_to_precompute( // frames would mint distinct sids and the upgrade // could never fire (the analyzer would also see two // candidates for one logical series). - let algorithm_for_sid = base_sketch_algorithm(sketch_algorithm_for(&dp)); - let agg_kind = crate::storage_engines::sketch_db::data::AggKind::Sketch { - algorithm: algorithm_for_sid, - config: dp.container_config.clone(), - // OTel-ingest path: no per-DP spatial filter applies - // at this layer (the agent has already filtered - // before emitting the sketch). The empty filter is - // the canonical value for "no filter on this sid's - // policy". - spatial_filter_canonical: String::new(), - }; - let agg_kind_canonical = match frame_identity.as_ref() { - Some(frame) => format!( - "{}|output:{}:{}:{}", - agg_kind.canonical_string(), - frame.plan_id, - frame.plan_version, - frame.materialization.as_u64() - ), - None => agg_kind.canonical_string(), - }; let definition = frame_identity .as_ref() .map(|frame| frame.materialization) .unwrap_or_else(|| asap_types::PolicyFingerprint(0).into()); - let assigned = match ingest_state.series_resolver.resolve_with_reactivation( - &canonical_name, + let resolved = ingest_state.summary_store.resolve_output_storage_handle( + &ingest_state.series_resolver, + definition, &fp, - &agg_kind_canonical, - |sid| { - ingest_state - .summary_store - .validate_routed_catalog_generation( - catalog_generation.as_deref(), - )?; - let activation = ingest_state - .summary_store - .authorize_series_reactivation(sid, definition)?; - if activation.as_deref().is_some_and(|generation| { - Some(generation) != catalog_generation.as_deref() - }) { - return Err( - "stale OTLP generation cannot reactivate series".into() - ); - } - Ok(activation) - }, - ) { + catalog_generation.as_deref(), + ); + // Codec-only unit fixtures have no installed plan. Production + // accepts stored summaries only through an installed output. + #[cfg(test)] + let resolved = if catalog_generation.is_none() + && definition.fingerprint().is_unset() + { + let kind = crate::storage_engines::sketch_db::data::AggKind::Sketch { + algorithm: base_sketch_algorithm(sketch_algorithm_for(&dp)), + config: dp.container_config.clone(), + spatial_filter_canonical: String::new(), + }; + Ok(ingest_state.series_resolver.resolve( + &canonical_name, + &fp, + &kind.canonical_string(), + )) + } else { + resolved + }; + let assigned = match resolved { Ok(sid) => sid, Err(error) => { if dp.series_id != 0 { @@ -1451,7 +1417,7 @@ async fn route_modified_otlp_sketches_to_precompute( .map(|s| s.to_string()) }; ingest_state.summary_store.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: canonical_name.clone(), group_by_keys, capability: Some(cap), @@ -3600,7 +3566,7 @@ mod sid_resolution_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -3613,8 +3579,8 @@ mod sid_resolution_tests { async fn make_state() -> (Arc, tokio::task::JoinHandle<()>) { let (tx, mut rx) = mpsc::channel(1024); let router = SeriesRouter::new(vec![tx]); - let streaming = StreamingConfig::new(std::collections::HashMap::new()); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(std::collections::HashMap::new()); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, samples_ingested: std::sync::atomic::AtomicU64::new(0), @@ -4536,7 +4502,7 @@ mod sid_bucketing_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -4655,8 +4621,8 @@ mod sid_bucketing_tests { let policy_fp = asap_types::PolicyFingerprint(cfg.policy_fp_u64()); let mut configs = HashMap::new(); configs.insert(cfg.policy_fp_u64(), cfg.clone()); - let streaming = StreamingConfig::new(configs); - let hot_reload = StreamingConfigHandle::new(streaming); + let streaming = InstalledPrecomputePlan::new(configs); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming); let resolver = Arc::new(SeriesIdResolver::new()); let state = Arc::new(IngestState { @@ -4751,7 +4717,10 @@ mod sid_bucketing_tests { }; let fp = crate::drivers::ingest::canonical_attrs_fingerprint(&[("zone", zone_for_bucket)]); - let resolved = resolver.lookup(metric, &fp, &agg_kind_canonical); + let resolved = state + .summary_store + .resolve_output_storage_handle(&resolver, policy_fp.into(), &fp, None) + .ok(); assert_eq!( resolved, Some(*sid), diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 3ddd40607..3816c0dbc 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -61,6 +61,10 @@ pub struct PrometheusRemoteWriteConfig { pub max_samples: usize, pub dedup_horizon: Duration, pub max_dedup_entries: usize, + pub revisions: Option<( + std::path::PathBuf, + crate::precompute_engine::revisions::RevisionPolicy, + )>, } impl Default for PrometheusRemoteWriteConfig { @@ -72,6 +76,7 @@ impl Default for PrometheusRemoteWriteConfig { max_samples: 1_000_000, dedup_horizon: Duration::from_secs(10 * 60), max_dedup_entries: 2_000_000, + revisions: None, } } } @@ -144,6 +149,8 @@ pub enum RemoteWriteError { InactivePhysicalPlan, #[error("series identity admission failed: {0}")] SeriesIdentity(String), + #[error("continuous revision execution failed: {0}")] + Revision(#[source] crate::precompute_engine::revisions::RevisionError), #[error(transparent)] Backpressure(#[from] TryRouteError), } @@ -161,6 +168,19 @@ pub struct CanonicalSample { impl PrometheusRemoteWriteReceiver { pub fn new(config: PrometheusRemoteWriteConfig, ingest: Arc) -> Self { + if let Some((directory, policy)) = &config.revisions { + *ingest + .summary_store + .revisions + .write() + .expect("revision installation poisoned") = Some(Arc::new( + crate::precompute_engine::revisions::RevisionRuntime::new( + directory.clone(), + policy.clone(), + ingest.hot_reload_config.clone(), + ), + )); + } Self { inner: Arc::new(ReceiverInner { config, @@ -190,6 +210,12 @@ impl PrometheusRemoteWriteReceiver { /// Permanently seal this finite source before queuing worker barriers. pub async fn drain(&self) -> Result<(), String> { + if self.inner.config.revisions.is_some() { + return Err( + "continuous input uses revisable snapshots; it cannot be sealed by a finite drain" + .into(), + ); + } { let mut state = self.inner.dedup.lock().map_err(|e| e.to_string())?; state.input_closed = true; @@ -233,11 +259,7 @@ impl PrometheusRemoteWriteReceiver { if plan.precompute_plan.summary_catalog.as_ref() != Some(&generation) { return Err("finite maintenance generation changed during drain".into()); } - crate::precompute_engine::maintenance_runtime::execute_finite_maintenance( - &self.inner.ingest.summary_store, - &self.inner.ingest.series_resolver, - &plan.precompute_plan, - )?; + self.inner.ingest.router.complete_dag(plan).await?; if let Some(observer) = self.inner.ingest.router.erp_observer() { let now_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -315,6 +337,23 @@ impl PrometheusRemoteWriteReceiver { if dedup.input_closed { return Err(RemoteWriteError::InputClosed); } + if let Some(runtime) = self + .inner + .ingest + .summary_store + .revisions + .read() + .map_err(|_| RemoteWriteError::Revision("revision installation poisoned".into()))? + .clone() + { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| RemoteWriteError::Revision(e.into()))? + .as_millis() as u64; + self.accept_revision(&runtime, samples, now_ms) + .map_err(RemoteWriteError::Revision)?; + return Ok(()); + } let batch_max_timestamp_ms = samples.iter().map(|sample| sample.timestamp_ms).max(); let max_event_timestamp_ms = match (dedup.max_event_timestamp_ms, batch_max_timestamp_ms) { (Some(current), Some(batch)) => Some(current.max(batch)), @@ -676,7 +715,7 @@ fn route_messages( Arc, ); type RoutedSample = (String, i64, f64); - let snapshot = physical_plan.streaming_config.clone(); + let snapshot = physical_plan.installed_precompute_plan.clone(); let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( ingest.summary_store.as_ref(), &snapshot, @@ -766,30 +805,14 @@ fn route_messages( &computed_attrs_fp }; let policy_fp = asap_types::PolicyFingerprint(config.policy_fp_u64()); - // A sketch family is not a complete physical identity. Two - // materializations may use the same family and grouping while - // differing in update semantics (for example count- versus - // value-weighted Top-K). Keep those states on distinct SIDs. - let materialization_kind = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); let sid = ingest - .series_resolver - .resolve_with_reactivation(&config.metric, attrs_fp, &materialization_kind, |sid| { - ingest.summary_store.validate_routed_catalog_generation( - physical_plan.precompute_plan.summary_catalog.as_ref(), - )?; - let activation = ingest - .summary_store - .authorize_series_reactivation(sid, policy_fp.into())?; - if let Some(generation) = &activation { - if physical_plan.precompute_plan.summary_catalog.as_ref() - != Some(generation.as_ref()) - { - return Err("stale routed generation cannot reactivate series".into()); - } - } - Ok(activation) - }) + .summary_store + .resolve_output_storage_handle( + &ingest.series_resolver, + policy_fp.into(), + attrs_fp, + physical_plan.precompute_plan.summary_catalog.as_ref(), + ) .map_err(RemoteWriteError::SeriesIdentity)?; buckets .entry(sid) @@ -936,8 +959,8 @@ mod tests { use crate::precompute_engine::ingest_handler::IngestObservability; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::{ - ActivePhysicalPlanHandle, BackendStorageRouting, RuntimePhysicalPlan, StreamingConfig, - StreamingConfigHandle, + ActivePhysicalPlanHandle, BackendStorageRouting, InstalledPrecomputePlan, + InstalledPrecomputePlanHandle, RuntimePhysicalPlan, }; use tokio::sync::mpsc; @@ -947,7 +970,7 @@ mod tests { .unwrap() } - fn physical_config(streaming: StreamingConfig) -> StreamingConfigHandle { + fn physical_config(streaming: InstalledPrecomputePlan) -> InstalledPrecomputePlanHandle { use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, @@ -1014,11 +1037,13 @@ mod tests { }, rules: Vec::new(), }, - streaming_config: Arc::new(streaming), + installed_precompute_plan: Arc::new(streaming), query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; - StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new(active)) + InstalledPrecomputePlanHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new( + active, + )) } fn receiver(config: PrometheusRemoteWriteConfig) -> PrometheusRemoteWriteReceiver { @@ -1027,7 +1052,7 @@ mod tests { router: SeriesRouter::new(vec![sender]), samples_ingested: AtomicU64::new(0), samples_blocked_by_schema_barrier: AtomicU64::new(0), - hot_reload_config: physical_config(StreamingConfig::default()), + hot_reload_config: physical_config(InstalledPrecomputePlan::default()), pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), @@ -1069,7 +1094,7 @@ mod tests { value_source_column: None, }; let policy_fp = aggregation.policy_fp_u64(); - let streaming = StreamingConfig::new(HashMap::from([(policy_fp, aggregation)])); + let streaming = InstalledPrecomputePlan::new(HashMap::from([(policy_fp, aggregation)])); let (sender, receiver) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -1107,7 +1132,7 @@ mod tests { let mut config = snapshot.precompute_plan.materializations[0].clone(); config.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; config.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); - let hot = physical_config(StreamingConfig::new(HashMap::from([( + let hot = physical_config(InstalledPrecomputePlan::new(HashMap::from([( config.policy_fp_u64(), config.clone(), )]))); @@ -1223,7 +1248,7 @@ mod tests { assert_ne!(kll_fp, pooled_kll_fp); let cms_fp = cms.policy_fingerprint(); let counter_fp = counter.policy_fingerprint(); - let streaming = StreamingConfig::new(HashMap::from([ + let streaming = InstalledPrecomputePlan::new(HashMap::from([ (cms_fp.0, cms), (counter_fp.0, counter), (kll_fp.0, kll), @@ -1389,7 +1414,9 @@ mod tests { router: SeriesRouter::new(vec![sender]), samples_ingested: AtomicU64::new(0), samples_blocked_by_schema_barrier: AtomicU64::new(0), - hot_reload_config: StreamingConfigHandle::new(StreamingConfig::default()), + hot_reload_config: InstalledPrecomputePlanHandle::new( + InstalledPrecomputePlan::default(), + ), pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), @@ -1698,3 +1725,257 @@ mod tests { ); } } + +impl PrometheusRemoteWriteReceiver { + fn accept_revision( + &self, + runtime: &crate::precompute_engine::revisions::RevisionRuntime, + samples: Vec, + now_ms: u64, + ) -> Result { + use crate::precompute_engine::revisions::{encode_state, InputSample, RevisionRecord}; + use crate::precompute_engine::{raw_dag::RawDagProgram, window_manager::WindowManager}; + use asap_types::sds::StoredOutputId; + let (plan, store) = runtime.installed()?; + let generation = plan + .precompute_plan + .summary_catalog + .as_ref() + .ok_or("revision catalog missing")?; + let max_window = plan + .precompute_plan + .materializations + .iter() + .map(|c| c.window_size.saturating_mul(1000)) + .max() + .unwrap_or(0); + let query_history = plan + .query_plan + .entries + .values() + .map(|entry| entry.instant.lookback_ms) + .max() + .unwrap_or(0); + let planned_retention = plan + .precompute_plan + .materializations + .iter() + .filter_map(|c| { + c.num_aggregates_to_retain + .map(|n| n.saturating_mul(c.slide_interval).saturating_mul(1000)) + }) + .max() + .unwrap_or(0); + let current_series_history = plan + .query_plan + .entries + .values() + .flat_map(|entry| entry.nodes.values()) + .filter_map(|node| match node { + asap_types::query_plan::QueryPlanNode::Logical { + operator: + asap_types::query_plan::residual::ResidualQueryOperator::CurrentSeries { + population, + .. + }, + .. + } => Some(population.lookback_ms), + _ => None, + }) + .max() + .unwrap_or(0); + let required_history = runtime + .policy + .correction_horizon_ms + .max(query_history) + .max(current_series_history) + .max(planned_retention); + let retain = max_window + .checked_add(required_history) + .and_then(|v| v.checked_add(runtime.policy.max_query_staleness_ms)) + .ok_or("revision retention overflow")?; + let inputs: Vec<_> = samples + .into_iter() + .map(|s| InputSample { + metric: s.metric.to_string(), + labels: s + .labels + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + series: s.series_key.to_string(), + timestamp_ms: s.timestamp_ms, + first_revision: 0, + value: s.value, + }) + .collect(); + let outputs = plan + .precompute_plan + .materializations + .iter() + .map(|c| StoredOutputId(c.policy_fp_u64())) + .collect(); + store.apply(generation, inputs, now_ms, retain, &outputs, |samples, horizon| { + // Event-time maxima and flush timers cannot admit an expired correction. + // Use the backend admission clock, and reject the entire request first. + if samples.iter().any(|s| s.timestamp_ms < 0 || s.timestamp_ms as u64 > now_ms || (s.timestamp_ms as u64) < now_ms.saturating_sub(horizon)) { + return Err(crate::precompute_engine::revisions::AdmissionRejected("Remote Write request exceeds the correction horizon or contains future input").into()); + } + for config in plan.precompute_plan.materializations.iter().filter(|c| c.derived_input.is_none()) { + let filter = compile_spatial_filter(&config.spatial_filter_normalized)?; + let program = RawDagProgram::from_plan(&plan.precompute_plan, config)?; + let updater = program.updater()?; + for sample in samples { + let labels = sample.labels.iter().map(|(k,v)|(k.clone(),v.clone())).collect(); + if sample.metric == config.metric && filter.matches(&labels) { + if let Some(value) = sample.value { + program.validate_sample(updater.as_ref(), value).map_err(|_| crate::precompute_engine::revisions::AdmissionRejected("sample is outside the bound summary kernel's input domain"))?; + } + } + } + } + Ok(()) + }, |inputs, revision, captured_at_ms, admitted, publish| { + if admitted { + let new = inputs.iter().filter(|s| s.first_revision == revision); + let numeric = new.clone().filter(|s|s.value.is_some()).count() as u64; + let stale = new.filter(|s|s.value.is_none()).count() as u64; + self.inner.stats.samples.fetch_add(numeric, Ordering::Relaxed); + self.inner.stats.stale_markers.fetch_add(stale, Ordering::Relaxed); + self.inner.ingest.samples_ingested.fetch_add(numeric, Ordering::Relaxed); + crate::precompute_engine::metrics::record_accepted_samples(numeric); + } + let canonical = revision_samples(inputs)?; + let committed = store.committed_outputs(revision)?; + let routed = route_messages(&canonical, &self.inner.ingest, &plan)?; + let mut frozen = Vec::new(); + let mut records: BTreeMap> = plan.precompute_plan.materializations.iter().filter(|c| c.derived_input.is_none()).map(|c|(StoredOutputId(c.policy_fp_u64()), Vec::new())).collect(); + for message in routed { + let WorkerMessage::GroupSamples { sid, policy_fp, group_key, mut samples, .. } = message else { return Err("revision routing returned non-sample input".into()); }; + let config = plan.precompute_plan.materializations.iter().find(|c| c.policy_fingerprint() == policy_fp).ok_or("revision raw configuration missing")?; + if committed.contains_key(&policy_fp.0) { continue; } + let program = RawDagProgram::from_plan(&plan.precompute_plan, config)?; + let manager = WindowManager::with_layout(config.window_size, config.slide_interval, config.pane_origin_ms, &config.window_layout); + let mut windows: BTreeMap<(u64,u64), Box> = BTreeMap::new(); + samples.sort_by(|a,b| (&a.0,a.1).cmp(&(&b.0,b.1))); + for (series,time,value) in samples { + for start in manager.stored_bucket_starts(time) { + let (start,end) = manager.stored_bucket_bounds(start); + if start < 0 || end < 0 || end as u64 > captured_at_ms || (start as u64) < captured_at_ms.saturating_sub(retain.saturating_sub(max_window)) { continue; } + let key = (start as u64,end as u64); + if let std::collections::btree_map::Entry::Vacant(entry) = windows.entry(key) { entry.insert(program.updater()?); } + program.apply(windows.get_mut(&key).unwrap().as_mut(), &series, value, time)?; + let bytes = windows.values().map(|u|u.memory_usage_bytes()).sum::(); + if bytes > runtime.policy.max_checkpoint_bytes { return Err(Box::new(asap_physical_operators::Error::MemoryLimit)); } + } + } + let reference = plan.installed_precompute_plan.stored_output_reference(policy_fp.into()).ok_or("revision raw binding missing")?; + let group = group_key.as_population_labels(); + let states: BTreeMap<_, Arc> = windows.into_iter().map(|(w,updater)|(w,Arc::from(updater.into_accumulator()))).collect(); + for ((start,end),state) in &states { + records.get_mut(&policy_fp.into()).unwrap().push(RevisionRecord { reference: reference.clone(), group: group.clone(), start_ms:*start, end_ms:*end, payload:encode_state(Arc::clone(state),program.family.clone())? }); + } + frozen.push(crate::storage_engines::sketch_db::index::FrozenExactWindows { stored_output_reference: reference, storage_handle: sid, definition:policy_fp.into(), generation: Arc::new(generation.clone()), group, windows:states, singleton_population_complete:true }); + if frozen.iter().flat_map(|f| f.windows.values()).map(|s|s.approx_memory_bytes()).sum::() > runtime.policy.max_checkpoint_bytes { return Err(Box::new(asap_physical_operators::Error::MemoryLimit)); } + } + // Recovery must consume the exact committed raw bytes, not another + // randomized sketch realization from replaying identical input. + for (output, result) in records { + if let Some(previous) = committed.get(&output.0) { + let config = plan.precompute_plan.materializations.iter().find(|c| c.policy_fp_u64() == output.0).ok_or("committed raw output missing")?; + let mut groups: BTreeMap, BTreeMap<(u64,u64),Arc>> = BTreeMap::new(); + for record in previous { groups.entry(record.group.clone()).or_default().insert((record.start_ms,record.end_ms),crate::precompute_engine::revisions::decode_state(record,config)?); } + for (group,windows) in groups { + frozen.push(crate::storage_engines::sketch_db::index::FrozenExactWindows { + stored_output_reference: plan.installed_precompute_plan.stored_output_reference(output).ok_or("committed raw binding missing")?, storage_handle: output.0, definition: output, generation:Arc::new(generation.clone()), group, windows, singleton_population_complete:true, + }); + } + } else { publish(output,result)?; } + } + crate::precompute_engine::maintenance_runtime::execute_revision_outputs(&plan, &frozen, revision, runtime.policy.max_checkpoint_bytes, publish) + }) + } +} + +impl PrometheusRemoteWriteReceiver { + /// Validate and restore the active generation before opening request serving. + pub fn recover_revisions(&self) -> Result<(), RemoteWriteError> { + if let Some(runtime) = self + .inner + .ingest + .summary_store + .revisions + .read() + .map_err(|_| RemoteWriteError::Revision("revision installation poisoned".into()))? + .clone() + { + if self.inner.ingest.active_physical_plan_snapshot().is_some() { + runtime.installed().map_err(RemoteWriteError::Revision)?; + } + } + Ok(()) + } + + /// Capture newly closed windows even when no later event arrives. This is + /// a revisable snapshot of locally accepted input, not an event-time watermark. + pub fn refresh_revisions(&self) -> Result<(), RemoteWriteError> { + let _admission = self + .inner + .dedup + .lock() + .map_err(|_| RemoteWriteError::Revision("revision admission poisoned".into()))?; + let runtime = self + .inner + .ingest + .summary_store + .revisions + .read() + .map_err(|_| RemoteWriteError::Revision("revision installation poisoned".into()))? + .clone(); + if let Some(runtime) = runtime { + if self.inner.ingest.active_physical_plan_snapshot().is_none() { + return Ok(()); + } + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| RemoteWriteError::Revision(e.into()))? + .as_millis() as u64; + self.accept_revision(&runtime, Vec::new(), now) + .map_err(RemoteWriteError::Revision)?; + } + Ok(()) + } +} + +pub(crate) fn revision_samples( + inputs: &[crate::precompute_engine::revisions::InputSample], +) -> Result, crate::precompute_engine::revisions::RevisionError> { + let canonical: Vec<_> = inputs + .iter() + .map(|s| { + let attrs: Vec<_> = s + .labels + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + Ok(CanonicalSample { + metric: Arc::from(s.metric.as_str()), + labels: Arc::new( + s.labels + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ), + series_key: Arc::from(s.series.as_str()), + population_key: Arc::from(format!( + "__asap_population__{}", + serde_json::to_string(&s.labels)? + )), + all_attrs_fingerprint: Arc::from(super::canonical_attrs_fingerprint(&attrs)), + timestamp_ms: s.timestamp_ms, + value: s.value, + }) + }) + .collect::>()?; + Ok(canonical) +} diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 3a33f3c83..1199e8867 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -134,7 +134,7 @@ pub struct HttpServer { /// `QueryEngine`) and extended via [`Self::with_query_engine`] — /// e.g. to plug in a `GorillaQueryEngine` for the cold archive /// tier. Instant-query dispatch consults this for metrics whose - /// `StreamingConfig::storage_backend()` is anything other than + /// `InstalledPrecomputePlan::storage_backend()` is anything other than /// `SketchStore`; ASAP-tier queries still take the direct /// `ASAPQueryEngine::handle_query` path so they keep the /// `KeyByLabelNames` Prometheus needs to populate the `metric` @@ -142,9 +142,9 @@ pub struct HttpServer { query_router: Arc, /// Sketch storage for runtime diagnostics. summary_store: Arc, - /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload + /// Hot-reloadable `InstalledPrecomputePlan` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). - hot_reload_config: Option, + hot_reload_config: Option, /// Per-metric storage-backend routing table consulted by the HTTP /// instant-query handler at request time. When `Some(..)` and the /// query parses, the handler extracts the metric name from the @@ -201,7 +201,7 @@ struct AppState { summary_store: Arc, adapter: Arc, fallback: Option>, - hot_reload_config: Option, + hot_reload_config: Option, /// See [`HttpServer::backend_storage_routing`]. backend_storage_routing: Option, /// Backfill registry (sketch DB §10). See `HttpServer::backfill`. @@ -289,13 +289,11 @@ impl HttpServer { self } - /// Attach a `StreamingConfigHandle` handle so the - /// `GET/POST /api/v1/streaming-config` endpoints can read and - /// swap the currently active config. Without this handle the - /// endpoints return `503 Service Unavailable`. + /// Attach the precompute projection used by installed-plan consumers. + /// Computation changes are published through the complete physical plan. pub fn with_hot_reload_config( mut self, - handle: crate::storage_engines::types::StreamingConfigHandle, + handle: crate::storage_engines::types::InstalledPrecomputePlanHandle, ) -> Self { self.hot_reload_config = Some(handle); self @@ -328,7 +326,7 @@ impl HttpServer { /// and dispatches through `EngineRouter` for any per-metric /// override. Without the wrapper the handler falls back to the /// pre-Phase-5 single-axis behaviour driven by - /// `StreamingConfig::storage_backend()`. + /// `InstalledPrecomputePlan::storage_backend()`. pub fn with_backend_storage_routing( mut self, routing: Arc, @@ -484,17 +482,6 @@ impl HttpServer { "/api/v1/db/backfill/jobs/:job_id", get(handle_get_backfill_job).delete(handle_delete_backfill_job), ); - // Legacy partial configuration is available only when the distributed - // profile explicitly supplies its hot-reload handle. The ASAPQuery - // profile exposes only generation-scoped physical-plan publication. - let app = if app_state.hot_reload_config.is_some() { - app.route( - "/api/v1/streaming-config", - get(handle_get_streaming_config).post(handle_post_streaming_config), - ) - } else { - app - }; let app = if adapter.query_language() == asap_types::QueryLanguage::MetricsQl { app.route( "/select/:tenant/prometheus/api/v1/query", @@ -610,14 +597,6 @@ impl HttpServer { "/api/v1/db/backfill/jobs/:job_id", get(handle_get_backfill_job).delete(handle_delete_backfill_job), ); - let app = if app_state.hot_reload_config.is_some() { - app.route( - "/api/v1/streaming-config", - get(handle_get_streaming_config).post(handle_post_streaming_config), - ) - } else { - app - }; let app = if adapter.query_language() == asap_types::QueryLanguage::MetricsQl { app.route( "/select/:tenant/prometheus/api/v1/query", @@ -783,7 +762,7 @@ async fn process_query_request( // AST and looked up in the table. This is the production // path the issue-46 MVP relies on so non-default metrics // actually route through the `EngineRouter`. - // (b) Single-axis `StreamingConfig::storage_backend()` from the + // (b) Single-axis `InstalledPrecomputePlan::storage_backend()` from the // hot-reload config (the pre-Phase-5 fallback). Pre-control-plane // deploys ride this path; it always lands on `SketchStore` // unless the YAML was hand-patched. @@ -961,12 +940,12 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag return backend; } debug!( - "resolve_metric_storage: PromQL parsed but no metric name found in AST; falling back to streaming-config axis", + "resolve_metric_storage: PromQL parsed but no metric name found in AST; falling back to installed storage axis", ); } Err(e) => { debug!( - "resolve_metric_storage: PromQL parse failed ({}); falling back to streaming-config axis", + "resolve_metric_storage: PromQL parse failed ({}); falling back to installed storage axis", e, ); } @@ -1383,7 +1362,7 @@ async fn process_via_named_engine( /// pinned by the request: /// /// * `query` — straight from the parsed request. -/// * `metric_storage` — looked up from the hot-reload `StreamingConfig` +/// * `metric_storage` — looked up from the hot-reload `InstalledPrecomputePlan` /// by the caller. /// /// `Statistic` remains a shape-derived follow-up. `AccuracyTarget` is @@ -2492,7 +2471,7 @@ mod tests { use crate::precompute_engine::ingest_handler::{IngestObservability, IngestState}; use crate::precompute_engine::series_router::SeriesRouter; use crate::query_engines::ASAPQueryEngine; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; use prost::Message; use reqwest::Client; use std::sync::atomic::AtomicU64; @@ -2509,7 +2488,7 @@ mod tests { IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, PLANNER_REVISION, }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let envelope = PlanEnvelope { plan_id: 7, plan_version: 1, @@ -2560,7 +2539,7 @@ mod tests { }, rules: Vec::new(), }, - streaming_config: streaming_config.clone(), + installed_precompute_plan: installed_precompute_plan.clone(), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id: 7, plan_version: 1, @@ -2573,7 +2552,7 @@ mod tests { ), }, ); - let hot_reload = StreamingConfigHandle::from_active_physical_plan(active.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_active_physical_plan(active.clone()); let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -2662,7 +2641,9 @@ mod tests { ); } - async fn setup_test_server_with_hot_reload(hot_reload: Option) -> u16 { + async fn setup_test_server_with_hot_reload( + hot_reload: Option, + ) -> u16 { let adapter_config = AdapterConfig::prometheus_promql( "http://127.0.0.1:9999".to_string(), // Unused for this test false, // forward_unsupported_queries @@ -2674,7 +2655,7 @@ mod tests { adapter_config, }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( @@ -2854,15 +2835,12 @@ mod tests { assert!(status.is_success() || status == reqwest::StatusCode::OK); } - // ── StreamingConfig hot-reload (PR E) ──────────────────────────────── + // ── InstalledPrecomputePlan hot-reload (PR E) ──────────────────────────────── - /// POST a YAML streaming-config and verify the active state via - /// GET reflects the swap. Covers the full round-trip through - /// `HttpServer::with_hot_reload_config`, the POST parse+swap, and - /// the GET snapshot emission. + /// A retired partial-configuration route cannot mutate the active plan. #[tokio::test] async fn flat_streaming_config_is_rejected_without_mutating_active_state() { - let handle = StreamingConfigHandle::new(StreamingConfig::default()); + let handle = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let port = setup_test_server_with_hot_reload(Some(handle.clone())).await; let before = handle.snapshot(); let response = Client::new() @@ -2871,7 +2849,7 @@ mod tests { .send() .await .unwrap(); - assert_eq!(response.status(), reqwest::StatusCode::GONE); + assert_eq!(response.status(), reqwest::StatusCode::NOT_FOUND); assert!(Arc::ptr_eq(&before, &handle.snapshot())); } @@ -2903,7 +2881,7 @@ mod tests { #[tokio::test] async fn test_streaming_config_hot_reload_rejects_bad_yaml() { - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -2915,9 +2893,7 @@ mod tests { .send() .await .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::GONE); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "error"); + assert_eq!(resp.status(), reqwest::StatusCode::NOT_FOUND); } /// Set up a test server wired with a hot-reload handle and a @@ -2928,7 +2904,7 @@ mod tests { /// legacy `SchemaRegistry` is gone, so there is no longer a /// `schemas` parameter — every reconcile decision is sid-level. async fn setup_test_server_with_hot_reload_and_sketch_index( - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, summary_store: Arc, ) -> u16 { let adapter_config = @@ -2938,7 +2914,7 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let server = HttpServer::new(config, query_engine, summary_store).with_hot_reload_config(hot_reload); @@ -2963,7 +2939,7 @@ mod tests { use std::collections::BTreeSet; let group_by_keys: BTreeSet = group_by.iter().map(|s| s.to_string()).collect(); store.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys, capability: None, @@ -2988,7 +2964,7 @@ mod tests { // handler force-retires it. use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let summary_store = Arc::new(SketchStore::new()); register_precompute_sid(&summary_store, 1, "m1", &[]); register_precompute_sid(&summary_store, 2, "m2", &[]); @@ -3018,11 +2994,11 @@ mod tests { assert_eq!(body["count"], 2); let entries = body["schemas"].as_array().unwrap(); // Sorted by sid — first is active, second is retired. - assert_eq!(entries[0]["sid"], 1); + assert_eq!(entries[0]["storage_handle"], 1); assert_eq!(entries[0]["status"], "active"); assert_eq!(entries[0]["metric_name"], "m1"); assert!(entries[0]["retired_at_ms"].is_null()); - assert_eq!(entries[1]["sid"], 2); + assert_eq!(entries[1]["storage_handle"], 2); assert_eq!(entries[1]["status"], "retired"); assert!(entries[1]["retired_at_ms"].is_u64()); @@ -3036,7 +3012,7 @@ mod tests { .unwrap(); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["count"], 1); - assert_eq!(body["schemas"][0]["sid"], 1); + assert_eq!(body["schemas"][0]["storage_handle"], 1); // Filter: retired only. let resp = client @@ -3048,7 +3024,7 @@ mod tests { .unwrap(); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["count"], 1); - assert_eq!(body["schemas"][0]["sid"], 2); + assert_eq!(body["schemas"][0]["storage_handle"], 2); // Bogus filter → 400. let resp = client @@ -3067,7 +3043,7 @@ mod tests { // attached (every `HttpServer` carries one). With no // registered sids the endpoint reports an empty array, not // a 503. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3093,7 +3069,7 @@ mod tests { use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::AggStatus; - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let summary_store = Arc::new(SketchStore::new()); register_precompute_sid(&summary_store, 11, "cpu", &["host"]); register_precompute_sid(&summary_store, 22, "mem", &["host"]); @@ -3115,7 +3091,7 @@ mod tests { assert!(resp.status().is_success()); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "success"); - assert_eq!(body["schema"]["sid"], 11); + assert_eq!(body["schema"]["storage_handle"], 11); assert_eq!(body["schema"]["status"], "retired"); assert_eq!( summary_store.instance(11).unwrap().status(), @@ -3133,7 +3109,7 @@ mod tests { assert!(resp.status().is_success()); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "success"); - assert_eq!(body["schema"]["sid"], 22); + assert_eq!(body["schema"]["storage_handle"], 22); assert_eq!(body["schema"]["status"], "expired"); assert_eq!( summary_store.instance(22).unwrap().status(), @@ -3158,7 +3134,7 @@ mod tests { async fn test_get_timeline_missing_param_returns_400() { use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let summary_store = Arc::new(SketchStore::new()); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), @@ -3204,7 +3180,7 @@ mod tests { // reads from the sid catalog (always attached) instead of the // optional `SchemaRegistry`. Empty catalog → empty segments, // not a 503. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); let resp = client @@ -3254,7 +3230,7 @@ mod tests { handle_http_requests: true, adapter_config, }; - // Build a StreamingConfig with one Sum agg per `active_agg_ids` + // Build a InstalledPrecomputePlan with one Sum agg per `active_agg_ids` // so the backfill handler's agg-config lookup (post-schema- // retirement) can find them. The matching sid in the catalog // is registered via the canonical ingest path so its content @@ -3298,8 +3274,8 @@ mod tests { marker_to_fp.insert(*marker, fp); agg_map.insert(fp, cfg); } - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - let hot_reload = StreamingConfigHandle::from_arc(streaming_config.clone()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::new(agg_map)); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(installed_precompute_plan.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); for marker in active_agg_ids { @@ -3458,7 +3434,7 @@ mod tests { async fn test_backfill_endpoints_503_without_registry() { // Build a server with NO backfill registry attached — every // backfill endpoint should 503. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3650,10 +3626,10 @@ mod tests { } /// Build an `HttpServer` whose router holds the supplied set of - /// `QueryEngine`s. The hot-reload `StreamingConfig` is pinned at + /// `QueryEngine`s. The hot-reload `InstalledPrecomputePlan` is pinned at /// `metric_storage_backend` so query dispatch follows the /// requested capability axis. Returns the bound port + the - /// `StreamingConfigHandle` handle so tests can swap the + /// `InstalledPrecomputePlanHandle` handle so tests can swap the /// `storage_backend` mid-flight if they need to. async fn setup_test_server_with_router( metric_storage_backend: StorageBackend, @@ -3679,10 +3655,12 @@ mod tests { }; // Pin `storage_backend` on the streaming config so the http // dispatcher reads it back through the hot-reload handle. - let streaming_cfg = - StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); + let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( + Default::default(), + metric_storage_backend, + ); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -3724,9 +3702,9 @@ mod tests { // — exactly what the production deploy looks like (the YAML // loader doesn't parse `storage_backend`). All routing // decisions must come from the per-metric routing table. - let streaming_cfg = StreamingConfig::default(); + let streaming_cfg = InstalledPrecomputePlan::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -3946,9 +3924,9 @@ mod tests { #[tokio::test] async fn http_query_with_no_storage_config_defaults_to_asap_tier() { - // `StreamingConfig::default()` has `storage_backend = + // `InstalledPrecomputePlan::default()` has `storage_backend = // SketchStore` (per the `#[serde(default)]` on the - // field — see `streaming_config.rs`). A server set up + // field — see `installed_precompute_plan.rs`). A server set up // without a hot-reload handle still infers ASAP-tier and // takes the ASAPQueryEngine direct path. Back-compat for // pre-Phase-5 deploys whose YAML doesn't include the new @@ -4186,10 +4164,8 @@ mod tests { // mock the routing decision by pinning `streaming_cfg.storage_backend // = GorillaObjectStore` directly. That proves the dispatch BRANCH is // wired, but not the production code path — in real deploys the - // streaming-config YAML loader drops `storage_backend` (it always - // defaults to `SketchStore`), so the issue-46 v2 demo's queries - // never reached the EngineRouter. The tests below exercise the - // **production path** end-to-end: streaming config stays default, + // installed precompute view uses the default storage backend. + // The tests below exercise per-metric routing end-to-end: // a per-metric `BackendStorageRouting` table is loaded at startup // (mirroring `--backend-storage-routing` on `precompute_engine`), // and the handler must consult the table on every request. @@ -4676,7 +4652,9 @@ mod tests { crate::query_engines::routing::HotReloadBackendStorageRouting, ) { use crate::query_engines::routing::HotReloadBackendStorageRouting; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{ + InstalledPrecomputePlan, InstalledPrecomputePlanHandle, + }; let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -4685,9 +4663,9 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_cfg = StreamingConfig::default(); + let streaming_cfg = InstalledPrecomputePlan::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let routing_handle = HotReloadBackendStorageRouting::empty(); let server = HttpServer::new( @@ -5007,10 +4985,12 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_cfg = - StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); + let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( + Default::default(), + metric_storage_backend, + ); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -5056,7 +5036,7 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_arc = Arc::new(StreamingConfig::default()); + let streaming_arc = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let cache = Arc::new(crate::query_engines::routing::FreshnessProbeCache::new()); let server = HttpServer::new( @@ -5278,7 +5258,14 @@ async fn handle_prometheus_remote_write( } } - match receiver.accept(&body) { + let receiver = receiver.clone(); + let accepted = match tokio::task::spawn_blocking(move || receiver.accept(&body)).await { + Ok(result) => result, + Err(error) => { + return (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response() + } + }; + match accepted { Ok(()) => StatusCode::NO_CONTENT.into_response(), Err(crate::drivers::ingest::prometheus_remote_write::RemoteWriteError::InputClosed) => (StatusCode::CONFLICT, "finite input is closed").into_response(), @@ -5304,6 +5291,10 @@ async fn handle_prometheus_remote_write( | crate::drivers::ingest::prometheus_remote_write::RemoteWriteError::DedupCapacity(_) | crate::drivers::ingest::prometheus_remote_write::RemoteWriteError::InactivePhysicalPlan), ) => (StatusCode::SERVICE_UNAVAILABLE, error.to_string()).into_response(), + Err(crate::drivers::ingest::prometheus_remote_write::RemoteWriteError::Revision(error)) => { + let status = if error.is::() { StatusCode::BAD_REQUEST } else { StatusCode::INTERNAL_SERVER_ERROR }; + (status, error.to_string()).into_response() + }, Err(error) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), } } @@ -5376,36 +5367,6 @@ async fn handle_store_metrics(State(state): State) -> axum::response:: (StatusCode::OK, axum::Json(body)).into_response() } -// Streaming configuration endpoints: GET returns the active snapshot; POST -// parses a replacement and atomically publishes it to new snapshot readers. -// In-flight operations retain their existing snapshot. - -async fn handle_get_streaming_config(State(state): State) -> axum::response::Response { - use axum::http::StatusCode; - use axum::response::IntoResponse; - - let Some(handle) = state.hot_reload_config else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload handle not attached; backend was built without HttpServer::with_hot_reload_config"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - let snap = handle.snapshot(); - let body = serde_json::json!({ - "status": "success", - "aggregation_count": snap.materializations_by_policy_fingerprint.len(), - "aggregation_ids": snap.materializations_by_policy_fingerprint.keys().copied().collect::>(), - "streaming_config": &*snap}); - (StatusCode::OK, axum::Json(body)).into_response() -} - -async fn handle_post_streaming_config() -> axum::response::Response { - use axum::response::IntoResponse; - (axum::http::StatusCode::GONE, axum::Json(serde_json::json!({ - "status":"error", "error":"install the complete DAG through /api/v1/physical-plan and activate its generation; partial aggregation config updates have been removed" - }))).into_response() -} - pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. @@ -5432,43 +5393,6 @@ pub fn validate_and_build_runtime_plan( .validate_against_catalog(&request.summary_catalog) .map_err(|error| format!("CollectorPlan catalog validation error: {error}"))?; } - for entry in request.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let materialization = request - .precompute_plan - .materializations - .iter() - .find(|config| config.policy_fingerprint() == binding.materialization.fingerprint()) - .ok_or_else(|| "query binding has no precompute definition".to_string())?; - if binding.window_ms != materialization.stored_window_ms() { - return Err( - "query physical pane duration differs from installed precompute definition" - .into(), - ); - } - if binding.pane_origin_ms != materialization.pane_origin_ms { - return Err( - "query physical pane origin differs from installed precompute definition" - .into(), - ); - } - // `full_window_slide_ms` is `#[serde(default)]`, so a publication from an - // older controller -- or one replayed from a stored artifact -- arrives as - // `None` on a FullWindow materialization. Without this gate the readout - // silently takes the overlap-merging path and counts observations twice, - // which is exactly what the full-window binding exists to prevent. - let full_window_slide_ms = matches!( - materialization.window_layout, - asap_types::WindowMaterializationLayout::FullWindow - ) - .then_some(materialization.slide_interval.saturating_mul(1_000)); - if binding.full_window_slide_ms != full_window_slide_ms { - return Err( - "query window layout differs from installed precompute definition".into(), - ); - } - } - } asap_types::plan_publication::validate_stored_output_references( &request.precompute_plan, &request.query_plan, @@ -5488,11 +5412,12 @@ pub fn validate_and_build_runtime_plan( { return Err("physical subplans have different plan identity/version".into()); } - let streaming_config = crate::storage_engines::types::StreamingConfig::from_precompute_plan( - request.precompute_plan.clone(), - ) - .map_err(|error| format!("DAG execution installation failed: {error}"))?; - let typed_fps: BTreeSet<_> = streaming_config + let installed_precompute_plan = + crate::storage_engines::types::InstalledPrecomputePlan::from_precompute_plan( + request.precompute_plan.clone(), + ) + .map_err(|error| format!("DAG execution installation failed: {error}"))?; + let typed_fps: BTreeSet<_> = installed_precompute_plan .materializations_by_policy_fingerprint .keys() .copied() @@ -5518,7 +5443,7 @@ pub fn validate_and_build_runtime_plan( summary_catalog: Some(Arc::new(request.summary_catalog)), precompute_plan: request.precompute_plan, transmission_plan: request.transmission_plan, - streaming_config: Arc::new(streaming_config), + installed_precompute_plan: Arc::new(installed_precompute_plan), query_plan: Arc::new(request.query_plan), storage_routing, }) @@ -5665,7 +5590,7 @@ async fn handle_activate_physical_plan( move |plan| match plan.summary_catalog.as_ref() { Some(catalog) => { store - .install_summary_catalog(Arc::clone(catalog)) + .install_precompute_plan(Arc::clone(catalog), &plan.precompute_plan) .map_err(|error| format!("SummaryCatalog install error: {error}"))?; if let Some(receiver) = &remote_write { receiver.install_erp_observation_generation( @@ -5709,7 +5634,7 @@ async fn handle_activate_physical_plan( } }); } - let snap = activated.streaming_config.clone(); + let snap = activated.installed_precompute_plan.clone(); let retired = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( state.summary_store.as_ref(), snap.as_ref(), @@ -5823,6 +5748,10 @@ async fn handle_physical_plan_status(State(state): State) -> axum::res ) .into_response(); }; + let precompute_plan = state + .active_physical_plan + .as_ref() + .map(|active| active.active_snapshot().precompute_plan.clone()); let materializations = state .active_physical_plan .as_ref() @@ -5833,6 +5762,7 @@ async fn handle_physical_plan_status(State(state): State) -> axum::res axum::Json(serde_json::json!({ "status": "success", "plans": lifecycle.statuses(), + "precompute_plan": precompute_plan, "materializations": materializations })), ) @@ -5853,9 +5783,7 @@ fn unix_time_ms() -> u64 { /// /// Useful for operators to confirm a control plane push landed with the /// expected entries. Returns 503 when the backend wasn't built with a -/// routing-table handle (legacy deploys that loaded the YAML directly -/// can still hit `/api/v1/streaming-config` — this endpoint is for -/// the Phase α JSON path). +/// routing-table handle. async fn handle_get_storage_routing( State(state): State, headers: axum::http::HeaderMap, @@ -5902,8 +5830,7 @@ async fn handle_get_storage_routing( /// * 503 — backend wasn't built with a routing-table handle. /// /// The swap is atomic: in-flight queries either see the entire old -/// table or the entire new table; never a half-applied state. Mirrors -/// the existing `POST /api/v1/streaming-config` swap contract. +/// table or the entire new table; never a half-applied state. async fn handle_post_storage_routing( State(state): State, headers: axum::http::HeaderMap, @@ -6012,11 +5939,23 @@ async fn handle_get_schemas( .iter() .filter(|m| allowed.contains(&m.status())) .map(|metadata| { - let descriptors = state.summary_store.descriptors_for_series_id(metadata.sid); - sid_instance_to_json(metadata, descriptors.as_ref()) + let descriptors = state + .summary_store + .descriptors_for_series_id(metadata.storage_handle); + stored_output_instance_to_json( + metadata, + descriptors.as_ref(), + state + .summary_store + .stored_output_for_handle(metadata.storage_handle), + ) }) .collect(); - entries.sort_by_key(|v| v.get("sid").and_then(|x| x.as_u64()).unwrap_or(0)); + entries.sort_by_key(|v| { + v.get("storage_handle") + .and_then(|x| x.as_u64()) + .unwrap_or(0) + }); let body = serde_json::json!({ "status": "success", @@ -6035,17 +5974,19 @@ fn status_str(s: crate::storage_engines::sketch_db::AggStatus) -> &'static str { } /// Encode sid metadata, including identity, lifecycle timestamps, and status. -fn sid_instance_to_json( +fn stored_output_instance_to_json( m: &crate::storage_engines::sketch_db::index::SummarySeriesMetadata, descriptors: Option<&( std::sync::Arc, std::sync::Arc, )>, + output: Option, ) -> serde_json::Value { let summary_descriptor_id = descriptors.map(|(summary, _)| summary.id.canonical()); let data_descriptor_id = descriptors.map(|(_, data)| data.id.canonical()); serde_json::json!({ - "sid": m.sid, + "storage_handle": m.storage_handle, + "stored_output_reference": output, "metric_name": m.metric_name, "status": status_str(m.status()), "first_seen_unix_ms": m.first_seen_unix_ms, @@ -6075,7 +6016,7 @@ async fn handle_post_schema_retire( let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", - "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); + "schema": stored_output_instance_to_json(&meta, descriptors.as_ref(), state.summary_store.stored_output_for_handle(meta.storage_handle))}); (StatusCode::OK, axum::Json(body)).into_response() } None => { @@ -6101,7 +6042,7 @@ async fn handle_post_schema_expire( let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", - "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); + "schema": stored_output_instance_to_json(&meta, descriptors.as_ref(), state.summary_store.stored_output_for_handle(meta.storage_handle))}); (StatusCode::OK, axum::Json(body)).into_response() } None => { @@ -6294,13 +6235,13 @@ async fn handle_post_backfill_job( return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); } - // Resolve the aggregation from the current streaming-config snapshot, or + // Resolve the aggregation from the current installed precompute plan, or // return 404 `UnknownAgg`. Its earliest matching sid timestamp bounds the // backfill range; with no ingested sid, use the current time. let Some(handle) = state.hot_reload_config.as_ref() else { let body = serde_json::json!({ "status": "error", - "error": "hot-reload streaming-config handle not attached; backfill agg lookup requires HttpServer::with_hot_reload_config"}); + "error": "hot-reload installed precompute handle not attached; backfill agg lookup requires HttpServer::with_hot_reload_config"}); return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); }; let snapshot = handle.snapshot(); @@ -6309,7 +6250,7 @@ async fn handle_post_backfill_job( None => { let body = serde_json::json!({ "status": "error", - "error": format!("unknown agg_id {} (not in streaming-config snapshot)", req.agg_id)}); + "error": format!("unknown agg_id {} (not in installed precompute plan)", req.agg_id)}); return (StatusCode::NOT_FOUND, axum::Json(body)).into_response(); } }; diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 2968fed2a..158d1acb0 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -59,8 +59,6 @@ pub use precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; pub use precompute_engine::output_sink::SketchStoreSink; pub use precompute_engine::PrecomputeEngine; -pub use utils::read_streaming_config; - pub type Result = std::result::Result>; #[cfg(test)] diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index e8db1fd1e..7061350cd 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -9,7 +9,6 @@ use data_plane::drivers::AdapterConfig; use data_plane::precompute_engine::config::LateDataPolicy; use data_plane::precompute_engine::PrecomputeWorkerDiagnostics; use data_plane::storage_engines::types::enums::{CleanupPolicy, LockStrategy}; -use data_plane::utils::file_io::read_streaming_config; use data_plane::{ ASAPQueryEngine, HttpServer, HttpServerConfig, OtlpReceiver, OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, PrometheusRemoteWriteConfig, @@ -41,9 +40,9 @@ struct Args { #[arg(long, value_enum, default_value = "distributed")] profile: RuntimeProfile, - /// Legacy bootstrap streaming config (distributed profile only). + /// Monitor coordinator settings, separate from executable computation plans. #[arg(long)] - streaming_config: Option, + monitor_specs: Option, /// JSON physical-plan artifact. Required by the backend-local profile; /// all runtime/query views are validated and installed as one snapshot. @@ -67,7 +66,7 @@ struct Args { /// the e2e harness's 30s window. ASAPQueryEngine uses this as the /// instant-query lookback window — for tumbling-window /// aggregations it must be ≥ the window size in - /// `streaming-config`. + /// the installed precompute plan. #[arg(long, default_value = "30")] prometheus_scrape_interval: u64, @@ -226,6 +225,16 @@ struct Args { #[arg(long, default_value = "2000000")] remote_write_max_dedup_entries: usize, + /// Enable bounded, revisable local Remote Write snapshots in this directory. + #[arg(long)] + remote_write_revision_dir: Option, + #[arg(long, default_value = "600000")] + remote_write_correction_horizon_ms: u64, + #[arg(long, default_value = "60000")] + remote_write_revision_freshness_ms: u64, + #[arg(long, default_value = "67108864")] + remote_write_revision_max_bytes: usize, + /// OTLP gRPC listen port #[arg(long, default_value = "4317")] otel_grpc_port: u16, @@ -235,7 +244,7 @@ struct Args { otel_http_port: u16, /// Enable the continuous-monitoring (CDM) coordinator gRPC server. Serves - /// the `monitors:` specs from the streaming-config; no-op if that list is + /// the specs from `--monitor-specs`; no-op if that list is /// empty. #[arg(long)] enable_monitor_coordinator: bool, @@ -378,7 +387,7 @@ struct Args { /// consulted by the HTTP query handler on every PromQL request to /// pick the right engine (`ASAPQueryEngine` for ASAP-tier /// sketches). Without - /// this flag the handler falls back to the streaming-config + /// this flag the handler falls back to the installed storage view /// single axis (always `SketchStore`) and the EngineRouter is /// effectively bypassed — the issue-46 v2 demo's criterion ⑤ /// failure mode. Mirrors the `precompute_engine` binary's flag @@ -421,21 +430,26 @@ fn validate_query_forwarding_configuration( fn validate_profile(args: &Args) -> Result<()> { validate_query_forwarding_configuration(args)?; + if args.remote_write_revision_dir.is_some() { + if !args.enable_remote_write && args.profile != RuntimeProfile::Asapquery { + return Err("--remote-write-revision-dir requires Remote Write ingestion".into()); + } + if args.remote_write_correction_horizon_ms == 0 + || args.remote_write_revision_freshness_ms == 0 + || args.remote_write_revision_max_bytes == 0 + { + return Err("continuous revision horizons and byte budget must be positive".into()); + } + } if args.profile != RuntimeProfile::Asapquery { - if args.streaming_config.is_none() { - return Err("the distributed profile requires --streaming-config".into()); + if args.physical_plan.is_none() { + return Err("the distributed profile requires --physical-plan".into()); } if args.planning_snapshot.is_some() { return Err("--planning-snapshot is available only with --profile asapquery".into()); } return Ok(()); } - if args.streaming_config.is_some() { - return Err( - "--profile asapquery rejects --streaming-config; use --planning-snapshot or --physical-plan" - .into(), - ); - } if args.physical_plan.is_some() == args.planning_snapshot.is_some() { return Err( "--profile asapquery requires exactly one of --planning-snapshot or --physical-plan" @@ -623,19 +637,12 @@ async fn main() -> Result<()> { } else { None }; - let streaming_config = match startup_physical_plan.as_ref() { - Some(active) => active.streaming_config.clone(), - None => Arc::new(read_streaming_config( - args.streaming_config - .as_deref() - .expect("validated distributed streaming config"), - )?), - }; + let startup_physical_plan = startup_physical_plan.ok_or("startup requires a physical plan")?; + let installed_precompute_plan = startup_physical_plan.installed_precompute_plan.clone(); info!( - "Loaded streaming config with {} entries", - streaming_config.materializations().len() + "Installed precompute DAG with {} stored outputs", + installed_precompute_plan.materializations().len() ); - info!("Streaming config: {:?}", streaming_config); // Share a hot-reload handle with HTTP configuration endpoints and consumers. @@ -672,12 +679,9 @@ async fn main() -> Result<()> { Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()) }; let summary_store = Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()); - if let Some(catalog) = startup_physical_plan - .as_ref() - .and_then(|plan| plan.summary_catalog.as_ref()) - { + if let Some(catalog) = startup_physical_plan.summary_catalog.as_ref() { summary_store - .install_summary_catalog(Arc::clone(catalog)) + .install_precompute_plan(Arc::clone(catalog), &startup_physical_plan.precompute_plan) .map_err(std::io::Error::other)?; } @@ -732,68 +736,11 @@ async fn main() -> Result<()> { None }; - // Bootstrap projections share one immutable physical-plan envelope. - let initial_precompute_plan = asap_types::precompute_plan::PrecomputePlan { - summary_catalog: None, - envelope: asap_types::precompute_plan::PlanEnvelope { - plan_id: 0, - plan_version: 0, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "bootstrap".into(), - planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), - capability_snapshot_id: "bootstrap".into(), - }, - ingest: asap_types::precompute_plan::IngestContract { - // Bootstrap envelope: no plan is installed yet, so there is no - // dataset to bind to. A published plan supplies the identity. - dataset_identity: None, - protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, - endpoint_path: "/v1/metrics".into(), - timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, - require_plan_identity: false, - require_stored_output_identity: false, - require_registered_producer: false, - }, - schemas: Vec::new(), - producers: Vec::new(), - materializations: streaming_config - .materializations_by_policy_fingerprint - .values() - .cloned() - .collect(), - executable_dags: Default::default(), - }; - let initial_transmission_plan = asap_types::producer_plan::TransmissionPlan { - summary_catalog: None, - envelope: initial_precompute_plan.envelope.clone(), - frame_identity: asap_types::producer_plan::FrameIdentityContract { - identity_version: 1, - sequence_scope: - asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, - require_checkpoint_for_full: true, - require_base_checkpoint_for_delta: true, - }, - rules: Vec::new(), - }; - let initial_active_plan = startup_physical_plan.unwrap_or_else(|| { - data_plane::storage_engines::types::RuntimePhysicalPlan { - envelope: initial_precompute_plan.envelope.clone(), - summary_catalog: None, - precompute_plan: initial_precompute_plan, - transmission_plan: initial_transmission_plan, - streaming_config: streaming_config.clone(), - query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), - storage_routing: Arc::new( - data_plane::storage_engines::types::BackendStorageRouting::empty(), - ), - } - }); + let initial_active_plan = startup_physical_plan; let active_physical_plan = data_plane::storage_engines::types::ActivePhysicalPlanHandle::new(initial_active_plan); let hot_reload_config = - data_plane::storage_engines::types::StreamingConfigHandle::from_active_physical_plan( + data_plane::storage_engines::types::InstalledPrecomputePlanHandle::from_active_physical_plan( active_physical_plan.clone(), ); @@ -975,69 +922,23 @@ async fn main() -> Result<()> { use data_plane::update_sampling::{ Functional, MonitorConfig, MonitorCoordinator, MonitorServiceImpl, }; - let specs: Vec = streaming_config - .monitors() - .iter() + let monitor_specs: Vec = match &args.monitor_specs { + Some(path) => serde_yaml::from_slice(&fs::read(path)?)?, + None => Vec::new(), + }; + let specs = monitor_specs + .into_iter() .map(|m| MonitorConfig { agg_id: m.agg_id, - key: m.key.clone().into_bytes(), + key: m.key.into_bytes(), tau: m.tau, epsilon: m.epsilon, window_ms: m.window_ms, functional: Functional::from_name(&m.functional), }) .collect(); - if specs.is_empty() { - warn!("--enable-monitor-coordinator set but streaming-config has no `monitors:` yet — the coordinator will pick them up live when the control plane pushes a config (hot-reload)"); - } let coord = MonitorCoordinator::new(specs); - // Hot-reload watcher: the coordinator reads `monitors:` once at boot, but - // the control plane pushes the real config slightly AFTER boot via the - // `/api/v1/streaming-config` POST (an ArcSwap in `hot_reload_config`). - // Without this, a monitor that arrives post-boot never reaches the - // coordinator and every edge registering for it is rejected as - // "unconfigured". Watch the ArcSwap and re-apply its `monitors:` to the - // live coordinator on each swap (cheap: an atomic load + pointer compare - // every 2s; `reconfigure` is a no-op unless the spec set actually - // changed). The same path covers controller-driven monitor add/remove. - { - let coord = coord.clone(); - let hot = hot_reload_config.clone(); - tokio::spawn(async move { - let mut last = hot.snapshot(); - loop { - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - let cur = hot.snapshot(); - if Arc::ptr_eq(&last, &cur) { - continue; - } - last = cur.clone(); - let specs: Vec = cur - .monitors() - .iter() - .map(|m| MonitorConfig { - agg_id: m.agg_id, - key: m.key.clone().into_bytes(), - tau: m.tau, - epsilon: m.epsilon, - window_ms: m.window_ms, - functional: Functional::from_name(&m.functional), - }) - .collect(); - let (added, changed, removed) = coord.reconfigure(specs).await; - if added + changed + removed > 0 { - info!( - added, - changed, - removed, - "CDM monitor coordinator hot-reloaded monitors from pushed streaming-config" - ); - } - } - }); - } - let svc = MonitorServiceImpl::new(coord).into_server(); let port = args.monitor_grpc_port; info!("Starting CDM monitor coordinator gRPC on 0.0.0.0:{port}"); @@ -1100,12 +1001,43 @@ async fn main() -> Result<()> { max_samples: args.remote_write_max_samples, dedup_horizon: std::time::Duration::from_millis(args.remote_write_dedup_horizon_ms), max_dedup_entries: args.remote_write_max_dedup_entries, + revisions: args.remote_write_revision_dir.clone().map(|directory| { + ( + directory, + data_plane::precompute_engine::revisions::RevisionPolicy { + correction_horizon_ms: args.remote_write_correction_horizon_ms, + max_query_staleness_ms: args.remote_write_revision_freshness_ms, + max_checkpoint_bytes: args.remote_write_revision_max_bytes, + }, + ) + }), }, precompute_ingest_state .clone() .expect("precompute ingest state is always constructed"), ); info!("Prometheus Remote Write v1 enabled at POST /api/v1/write"); + if args.remote_write_revision_dir.is_some() { + receiver.recover_revisions()?; + let continuous = receiver.clone(); + let capture_interval_ms = (args.remote_write_revision_freshness_ms / 2).clamp(1, 1000); + tokio::spawn(async move { + let mut tick = + tokio::time::interval(std::time::Duration::from_millis(capture_interval_ms)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tick.tick().await; + let receiver = continuous.clone(); + match tokio::task::spawn_blocking(move || receiver.refresh_revisions()).await { + Ok(Ok(())) => {} + outcome => { + tracing::error!(?outcome, "continuous revision execution stopped"); + break; + } + } + } + }); + } server = server.with_remote_write(receiver); } @@ -1114,7 +1046,7 @@ async fn main() -> Result<()> { // from `--backend-storage-routing` (or its env-var alias) so the // HTTP handler consults a per-metric `StorageBackend` map on // every PromQL query instead of bypassing the EngineRouter when - // the streaming-config single axis defaults to `SketchStore`. + // the installed storage view single axis defaults to `SketchStore`. // // even when no static YAML is loaded, install an // empty hot-reload handle so the control plane's first @@ -1156,7 +1088,7 @@ async fn main() -> Result<()> { summary_catalog: current.summary_catalog.clone(), precompute_plan: current.precompute_plan.clone(), transmission_plan: current.transmission_plan.clone(), - streaming_config: current.streaming_config.clone(), + installed_precompute_plan: current.installed_precompute_plan.clone(), query_plan: current.query_plan.clone(), storage_routing: Arc::new(bootstrap_routing), }); @@ -1187,7 +1119,7 @@ async fn main() -> Result<()> { args.enable_backfill_worker, precompute_ingest_state.as_ref(), ) { - // Backfill uses the shared streaming-config snapshot and sketch store. + // Backfill uses the installed precompute plan and sketch store. let reader_factory = match args.clickhouse_backfill_table.as_ref() { Some(table) => data_plane::storage_engines::sketch_db::clickhouse_reader_factory( data_plane::storage_engines::sketch_db::ClickHouseReaderConfig { @@ -1491,6 +1423,13 @@ mod tests { use clap::Parser; use data_plane::drivers::AdapterConfig; + // Every profile must bootstrap from the same validated physical artifact. + #[test] + fn distributed_accepts_physical_plan_without_streaming_config() { + let args = Args::try_parse_from(["data_plane", "--physical-plan", "plan.json"]).unwrap(); + assert!(validate_profile(&args).is_ok()); + } + #[test] fn asapquery_requires_atomic_physical_plan_not_streaming_config() { let valid = Args::try_parse_from([ @@ -1504,21 +1443,9 @@ mod tests { .unwrap(); assert!(validate_profile(&valid).is_ok()); - let legacy = Args::try_parse_from([ - "data_plane", - "--profile", - "asapquery", - "--streaming-config", - "streaming.yaml", - "--physical-plan", - "plan.json", - "--forward-unsupported-queries", - ]) - .unwrap(); - assert!(validate_profile(&legacy) - .unwrap_err() - .to_string() - .contains("rejects --streaming-config")); + assert!( + Args::try_parse_from(["data_plane", "--streaming-config", "streaming.yaml"]).is_err() + ); let planned = Args::try_parse_from([ "data_plane", @@ -1576,7 +1503,7 @@ mod tests { fn disable_query_forwarding_rejects_conflicting_flags() { let args = Args::try_parse_from([ "data_plane", - "--streaming-config", + "--physical-plan", "streaming.yaml", "--disable-query-forwarding", "--forward-unsupported-queries", @@ -1592,7 +1519,7 @@ mod tests { fn disable_query_forwarding_rejects_forwarding_listeners() { let args = Args::try_parse_from([ "data_plane", - "--streaming-config", + "--physical-plan", "streaming.yaml", "--disable-query-forwarding", "--victoriametrics-http-port", @@ -1609,7 +1536,7 @@ mod tests { fn disable_query_forwarding_rejects_clickhouse_listener() { let clickhouse = Args::try_parse_from([ "data_plane", - "--streaming-config", + "--physical-plan", "streaming.yaml", "--disable-query-forwarding", "--clickhouse-http-port", diff --git a/data_plane/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs index fa451ca36..d6b912e8a 100644 --- a/data_plane/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -3,7 +3,7 @@ use crate::precompute_engine::ingest_handler::IngestState; use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::precompute_engine::worker::{Worker, WorkerRuntimeConfig}; -use crate::storage_engines::types::StreamingConfigHandle; +use crate::storage_engines::types::InstalledPrecomputePlanHandle; use std::sync::atomic::{AtomicI64, AtomicUsize}; use std::sync::Arc; use tokio::sync::mpsc; @@ -27,7 +27,7 @@ pub struct PrecomputeEngine { output_sink: Arc, diagnostics: Arc, ingest_state: Arc, - hot_reload_config: StreamingConfigHandle, + hot_reload_config: InstalledPrecomputePlanHandle, /// Worker receivers, one per worker. Taken by `run()` when spawning workers. receivers: Vec>, } @@ -35,7 +35,7 @@ pub struct PrecomputeEngine { impl PrecomputeEngine { pub fn new( config: PrecomputeEngineConfig, - hot_reload_config: StreamingConfigHandle, + hot_reload_config: InstalledPrecomputePlanHandle, output_sink: Arc, series_resolver: Arc, summary_store: Arc, @@ -63,7 +63,7 @@ impl PrecomputeEngine { } // Build the router that owns the senders; it will be shared via IngestState. - let router = SeriesRouter::new(senders); + let router = SeriesRouter::new(senders).with_plan(hot_reload_config.clone()); // Snapshot the hot-reload configuration on each ingest batch so policy // changes are visible immediately. Sid lifecycle reconciliation uses @@ -107,13 +107,6 @@ impl PrecomputeEngine { /// submit through the shared `IngestState` returned by `ingest_state()`. pub async fn run(mut self) -> Result<(), Box> { let num_workers = self.config.num_workers; - let output_sink: Arc = Arc::new( - crate::precompute_engine::maintenance_runtime::MaintenanceDagSink::new( - Arc::clone(&self.output_sink), - self.hot_reload_config.clone(), - ), - ); - // Take ownership of receivers (they can only be used once). let receivers = std::mem::take(&mut self.receivers); @@ -122,6 +115,13 @@ impl PrecomputeEngine { // ConfigReload messages needed. let mut worker_handles = Vec::with_capacity(num_workers); for (id, rx) in receivers.into_iter().enumerate() { + let output_sink: Arc = Arc::new( + crate::precompute_engine::maintenance_runtime::MaintenanceDagSink::new( + Arc::clone(&self.output_sink), + self.hot_reload_config.clone(), + ), + ); + let mut worker = Worker::new( id, rx, @@ -141,6 +141,10 @@ impl PrecomputeEngine { self.diagnostics.worker_group_counts[id].clone(), self.diagnostics.worker_watermarks[id].clone(), ); + worker.set_maintenance_storage( + Arc::clone(&self.ingest_state.summary_store), + Arc::clone(&self.ingest_state.series_resolver), + ); worker.set_erp_observer(self.ingest_state.router.erp_observer()); let handle = tokio::spawn(async move { worker.run().await; @@ -155,7 +159,7 @@ impl PrecomputeEngine { // Start flush timer — pure flush, no config polling. let flush_state = ingest_state.clone(); let flush_interval_ms = self.config.flush_interval_ms; - tokio::spawn(async move { + let flush_task = tokio::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(flush_interval_ms)); loop { @@ -168,10 +172,16 @@ impl PrecomputeEngine { }); // Wait for workers to finish (this only happens on shutdown). + let mut worker_error = None; for handle in worker_handles { - let _ = handle.await; + if let Err(error) = handle.await { + worker_error = Some(error); + } + } + flush_task.abort(); + if let Some(error) = worker_error { + return Err(Box::new(error)); } - Ok(()) } } diff --git a/data_plane/src/precompute_engine/group_key.rs b/data_plane/src/precompute_engine/group_key.rs index 442122782..dd3c56842 100644 --- a/data_plane/src/precompute_engine/group_key.rs +++ b/data_plane/src/precompute_engine/group_key.rs @@ -45,7 +45,6 @@ impl GroupKey { } } - #[cfg(test)] pub fn canonical_bytes(&self) -> &[u8] { &self.canonical } diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index a531f3c0c..cf033c140 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,6 +1,6 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::StreamingConfigHandle; +use crate::storage_engines::types::InstalledPrecomputePlanHandle; use asap_types::aggregation_config::PrecomputeMaterialization; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -121,7 +121,7 @@ pub struct IngestState { /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. /// This replaces the old frozen `Vec>`. - pub hot_reload_config: StreamingConfigHandle, + pub hot_reload_config: InstalledPrecomputePlanHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, /// Per-series reconstructed sketch bases, keyed by series identity. Full frames @@ -156,10 +156,10 @@ impl IngestState { /// configs from a `POST /api/v1/streaming-config` swap are /// visible immediately without restart. /// - /// Returns the shared `Arc` — no cloning of + /// Returns the shared `Arc` — no cloning of /// individual PrecomputeMaterialization objects, just an atomic refcount /// increment (~5ns). - pub fn config_snapshot(&self) -> Arc { + pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() } @@ -293,7 +293,7 @@ fn extract_group_key( mod tests { use super::*; use crate::precompute_engine::series_router::SeriesRouter; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; @@ -336,9 +336,9 @@ mod tests { let mut map = std::collections::HashMap::new(); map.insert(agg_id, make_config(agg_id, metric)); - let streaming = StreamingConfig::new(map); + let streaming = InstalledPrecomputePlan::new(map); let hot_reload = - crate::storage_engines::types::StreamingConfigHandle::new(streaming.clone()); + crate::storage_engines::types::InstalledPrecomputePlanHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index bbed2b039..a48154c87 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -2,21 +2,42 @@ use super::output_sink::OutputSink; use super::subdag_scheduler::{ - execute_precompute_sink, IdempotentCommitSink, MaterializationCommitKey, - PrecomputeOperatorRegistry, ScheduleError, + execute_precompute_sink, execute_precompute_sinks, IdempotentCommitSink, + MaterializationCommitKey, PrecomputeOperatorRegistry, ScheduleError, }; -use crate::storage_engines::types::{AggregateCore, PrecomputedOutput, StreamingConfigHandle}; +use crate::storage_engines::types::{ + AggregateCore, InstalledPrecomputePlanHandle, PrecomputedOutput, +}; +use asap_physical_operators::dag::RunContext; use asap_types::executable_plan::{BackendExecutableBinding, BackendNodeBinding}; use planner_types::post_asap::{ExecutableDagNode, ExecutableOperatorPayload, PostAsapNodeId}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex}; +type MaintenanceError = Box; + type SummaryState = Arc; type Population = BTreeMap; type PopulationStates = BTreeMap>; type PopulationRows = BTreeMap>; +fn maintenance_context( + start: i64, + end: i64, + revision: u64, +) -> Result { + RunContext::new( + asap_physical_operators::dag::Scope::Ingestion { + window_start_ms: start, + window_end_ms: end, + revision, + }, + asap_physical_operators::dag::Limits::default(), + ) + .map_err(Into::into) +} + #[derive(Clone)] enum MaintenanceValue { Summary { @@ -44,7 +65,7 @@ impl MaintenanceValue { } } - fn state(&self) -> Result<&SummaryState, String> { + fn state(&self) -> Result<&SummaryState, MaintenanceError> { match self { Self::Summary { state, .. } => Ok(state), Self::SummaryWindows { states, .. } @@ -75,6 +96,7 @@ enum MaintenanceInputs<'a> { }, Frozen(&'a [crate::storage_engines::sketch_db::index::FrozenExactWindows]), Complete(&'a crate::storage_engines::sketch_db::index::CompleteRawMaintenanceCohort), + Captured(&'a [crate::storage_engines::sketch_db::index::FrozenExactWindows]), } impl MaintenanceInputs<'_> { @@ -83,7 +105,7 @@ impl MaintenanceInputs<'_> { ) -> Option<&[crate::storage_engines::sketch_db::index::FrozenExactWindows]> { match self { Self::Live { .. } => None, - Self::Frozen(inputs) => Some(inputs), + Self::Frozen(inputs) | Self::Captured(inputs) => Some(inputs), Self::Complete(cohort) => Some(cohort.inputs()), } } @@ -94,7 +116,7 @@ fn frozen_population_value( definition: asap_types::sds::StoredOutputId, family: Option, complete: bool, -) -> Result, String> { +) -> Result, MaintenanceError> { let mut states = PopulationStates::new(); for input in inputs.iter().filter(|input| input.definition == definition) { if !complete && !states.is_empty() { @@ -126,12 +148,12 @@ struct OperatorAdapter<'a> { } impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { - type Error = String; + type Error = MaintenanceError; fn materialized_input( &self, node: &ExecutableDagNode, - ) -> Result, String> { + ) -> Result, MaintenanceError> { let definition = match self.binding.node(node.id) { Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Ok(None), @@ -155,22 +177,54 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { MaintenanceInputs::Frozen(inputs) => { frozen_population_value(inputs, definition, family, false) } + MaintenanceInputs::Captured(inputs) => { + frozen_population_value(inputs, definition, family, true) + } MaintenanceInputs::Complete(cohort) => { frozen_population_value(cohort.inputs(), definition, family, true) } } } + fn output_bytes(&self, value: &MaintenanceValue) -> usize { + fn labels(group: &Population) -> usize { + group.iter().map(|(k, v)| k.len() + v.len()).sum() + } + match value { + MaintenanceValue::Summary { state, .. } => state.approx_memory_bytes(), + MaintenanceValue::SummaryWindows { states, .. } => states + .iter() + .map(|(group, windows)| { + labels(group) + + windows + .iter() + .map(|(_, state)| 8 + state.approx_memory_bytes()) + .sum::() + }) + .sum(), + MaintenanceValue::Rows { values, name, .. } => { + name.len() + + values + .iter() + .map(|(group, rows)| { + labels(group) + rows.len() * std::mem::size_of::<(i64, f64)>() + }) + .sum::() + } + } + } + fn execute( &self, node: &ExecutableDagNode, inputs: &[Arc], + context: RunContext, ) -> Result { if node.output_state.timing != planner_types::post_asap::ExecutionTiming::IngestionTime { return Err("ingestion executor received a query-time node".into()); } match &node.payload { - ExecutableOperatorPayload::SummaryMerge => merge_inputs(inputs), + ExecutableOperatorPayload::SummaryMerge => merge_inputs(inputs, &context), ExecutableOperatorPayload::Binary { operator } => { if !self.inputs.frozen_inputs().is_some() || node.output_state @@ -178,7 +232,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { { return Err("maintenance binary requires immutable completed row inputs".into()); } - evaluate_aligned_binary(node, operator, inputs) + evaluate_aligned_binary(node, operator, inputs, &context) } ExecutableOperatorPayload::Value { @@ -190,7 +244,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { .into(), ); } - finalize_exact(node, inputs) + finalize_exact(node, inputs, &context) } ExecutableOperatorPayload::SummaryAgg { family, @@ -243,7 +297,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { || matches!(&self.inputs, MaintenanceInputs::Complete(_)), )?; } - if config.accumulator_spec().map_err(|e| e.to_string())?.family != *family { + if config.accumulator_spec()?.family != *family { return Err( "maintenance SummaryAgg family differs from installed configuration".into(), ); @@ -253,12 +307,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "keyed maintenance updates require explicit row identity routing".into(), ); } - let mut updater = asap_physical_operators::factory::create_planner_accumulator( - family, input, grouping, - )?; - if updater.is_keyed() { - return Err("keyed maintenance accumulator requires an item expression".into()); - } + let _ = grouping; let output_groups = values .keys() .map(|group| { @@ -288,11 +337,38 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "maintenance sink requires one explicitly reduced output population".into(), ); } - for (timestamp_ms, value) in values.values().flatten() { - let weight = evaluate_weight(&input.weight, *value, name)?; - updater.validate_single_input(weight)?; - updater.update_single(weight, *timestamp_ms); - } + use asap_physical_operators::dag::{operators::Operator, values::Value}; + let schema = native_schema(vec![ + ( + "value", + planner_types::post_asap::SummaryFamilyType::Plain( + planner_types::pre_asap::DataType::Float64, + ), + ), + ( + "time", + planner_types::post_asap::SummaryFamilyType::Plain( + planner_types::pre_asap::DataType::Timestamp, + ), + ), + ]); + let rows = values + .values() + .flatten() + .map(|(time, value)| { + Ok(vec![ + Value::Float64(evaluate_weight(&input.weight, *value, name)?), + Value::Timestamp(*time), + ]) + }) + .collect::, MaintenanceError>>()?; + let builder = + Operator::summary_build(schema.clone(), family.clone(), 0, Some(1), vec![])?; + let mut result = native_rows(schema, rows, vec![builder], &context)?; + let Some(Value::Summary { state, .. }) = result.pop().and_then(|mut row| row.pop()) + else { + return Err("native summary builder did not return state".into()); + }; let timestamp = values .values() .flatten() @@ -302,24 +378,109 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { Ok(MaintenanceValue::SummaryWindows { states: BTreeMap::from([( output_groups.into_iter().next().unwrap(), - vec![(timestamp, Arc::from(updater.into_accumulator()))].into(), + vec![(timestamp, state)].into(), )]), family: family.clone(), }) } payload => Err(format!( "maintenance operator {payload:?} has no summary-state implementation" - )), + ) + .into()), } } } +// These functions translate deployment values into native batches. Window and +// catalog checks stay here; the library owns all computation on batch values. +fn native_schema( + fields: Vec<(&str, planner_types::post_asap::SummaryFamilyType)>, +) -> asap_physical_operators::dag::values::Schema { + Arc::new(planner_types::post_asap::SummarySchema { + fields: fields + .into_iter() + .map(|(name, dtype)| planner_types::post_asap::SummaryField { + name: name.into(), + dtype, + nullable: false, + }) + .collect(), + time_index: None, + }) +} +fn native_rows( + schema: asap_physical_operators::dag::values::Schema, + rows: Vec>, + operators: Vec, + context: &RunContext, +) -> Result>, MaintenanceError> { + use asap_physical_operators::dag::{batch_execution::evaluate_batch, values::Batch}; + let input = Batch::try_new(schema, rows)?; + Ok(evaluate_batch(input, operators, context.clone())? + .into_iter() + .flat_map(|b| b.rows().to_vec()) + .collect()) +} +fn native_arithmetic( + op: &planner_types::post_asap::BinaryOperator, + inputs: Vec<(i64, f64, f64)>, + context: &RunContext, +) -> Result, MaintenanceError> { + use asap_physical_operators::dag::{ + operators::{Expression, Operator}, + values::Value, + }; + use planner_types::{post_asap::SummaryFamilyType, pre_asap::DataType}; + let schema = native_schema(vec![ + ("time", SummaryFamilyType::Plain(DataType::Timestamp)), + ("left", SummaryFamilyType::Plain(DataType::Float64)), + ("right", SummaryFamilyType::Plain(DataType::Float64)), + ]); + let project = Operator::project( + schema.clone(), + vec![ + ("time".into(), Expression::Column(0)), + ( + "value".into(), + Expression::Binary { + operator: op.clone(), + left: Box::new(Expression::Column(1)), + right: Box::new(Expression::Column(2)), + }, + ), + ], + )?; + let rows = native_rows( + schema, + inputs + .into_iter() + .map(|(time, left, right)| { + vec![ + Value::Timestamp(time), + Value::Float64(left), + Value::Float64(right), + ] + }) + .collect(), + vec![project], + context, + )?; + rows.into_iter() + .map(|row| match row.as_slice() { + [Value::Timestamp(time), Value::Float64(value)] if value.is_finite() => { + Ok((*time, *value)) + } + _ => Err("maintenance binary produced a non-finite update".into()), + }) + .collect() +} + fn validate_maintenance_grouping( target: &asap_types::PrecomputeMaterialization, source: &asap_types::PrecomputeMaterialization, node: &ExecutableDagNode, singleton_population_complete: bool, -) -> Result<(), String> { +) -> Result<(), MaintenanceError> { if target.grouping_labels == source.grouping_labels && target.partitioning == source.partitioning { @@ -343,7 +504,7 @@ fn evaluate_weight( expression: &planner_types::post_asap::SummaryInputExpr, value: f64, name: &str, -) -> Result { +) -> Result { use planner_types::{post_asap::SummaryInputExpr, pre_asap::ColumnRef}; let weight = match expression { SummaryInputExpr::Constant(value) => *value, @@ -363,7 +524,7 @@ fn evaluate_weight( // would require silently dropping another column or manufacturing a timestamp. fn maintenance_float64_column( node: &ExecutableDagNode, -) -> Result<&planner_types::post_asap::SummaryField, String> { +) -> Result<&planner_types::post_asap::SummaryField, MaintenanceError> { use planner_types::post_asap::SummaryFamilyType; let fields = &node.output_schema.fields; let field = match node.output_schema.time_index { @@ -406,9 +567,10 @@ fn evaluate_aligned_binary( node: &ExecutableDagNode, operator: &planner_types::post_asap::BinaryOperator, inputs: &[Arc], -) -> Result { + context: &RunContext, +) -> Result { use planner_types::pre_asap::BinaryOpKind; - let BinaryOpKind::Arithmetic(arithmetic) = &operator.kind else { + let BinaryOpKind::Arithmetic(_) = &operator.kind else { return Err("maintenance binary currently requires arithmetic".into()); }; if operator.vector_match.is_some() { @@ -466,14 +628,9 @@ fn evaluate_aligned_binary( let right = right_rows .get(×tamp) .ok_or("maintenance binary requires matching timestamp sets")?; - let value = asap_physical_operators::arithmetic::evaluate_float64_arithmetic( - arithmetic, left, *right, - ); - if !value.is_finite() { - return Err("maintenance binary produced a non-finite update".into()); - } - joined.push((timestamp, value)); + joined.push((timestamp, left, *right)); } + let joined = native_arithmetic(operator, joined, context)?; values.insert(group.clone(), joined); } Ok(MaintenanceValue::Rows { @@ -486,7 +643,8 @@ fn evaluate_aligned_binary( fn finalize_exact( node: &ExecutableDagNode, inputs: &[Arc], -) -> Result { + context: &RunContext, +) -> Result { use planner_types::post_asap::{ExactKind, SummaryFamilyType}; let [input] = inputs else { return Err("exact maintenance finalization requires one summary input".into()); @@ -532,12 +690,38 @@ fn finalize_exact( let field = maintenance_float64_column(node)?; let mut values = PopulationRows::new(); for (group, states) in groups { + use asap_physical_operators::dag::{operators::Operator, values::Value}; + let schema = native_schema(vec![("state", family.clone())]); + let readout = Operator::readout(schema.clone(), 0, statistic, Default::default())?; + let rows = native_rows( + schema, + states + .iter() + .map(|(_, state)| { + vec![Value::Summary { + family: family.clone(), + state: Arc::clone(state), + }] + }) + .collect(), + vec![readout], + context, + )?; let rows = states .into_iter() - .map(|(timestamp, state)| { - let value = state - .query_statistic(statistic, &None, &std::collections::HashMap::new()) - .map_err(|error| error.to_string())?; + .zip(rows) + .map(|((timestamp, _), row)| { + let value = + match row.first() { + Some(Value::Float64(value)) => *value, + Some(Value::Int64(value)) if value.unsigned_abs() <= (1u64 << 53) => { + *value as f64 + } + _ => return Err( + "exact readout cannot be represented by the installed Float64 schema" + .to_string(), + ), + }; if !value.is_finite() { return Err("exact maintenance finalization produced a non-finite value".into()); } @@ -554,7 +738,10 @@ fn finalize_exact( }) } -fn merge_inputs(inputs: &[Arc]) -> Result { +fn merge_inputs( + inputs: &[Arc], + context: &RunContext, +) -> Result { let mut grouped = BTreeMap::, Option)>::new(); let mut expected_groups = None; let mut family = None; @@ -611,19 +798,35 @@ fn merge_inputs(inputs: &[Arc]) -> Result().map(|state| state.family().clone()) + }).or_else(|| first.as_any().is::().then_some( + planner_types::post_asap::SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, planner_types::post_asap::ExactParams::Sum) + )).ok_or("summary merge requires a registered state family")?; + use asap_physical_operators::dag::{operators::Operator, values::Value}; + let schema = native_schema(vec![("state", state_family.clone())]); + let rows = std::iter::once(*first) + .chain(rest.iter().copied()) + .map(|state| { + vec![Value::Summary { + family: state_family.clone(), + state: Arc::clone(state), + }] + }) + .collect(); + let merge = Operator::summary_merge(schema.clone(), 0, vec![])?; + let mut output = native_rows(schema, rows, vec![merge], context)?; + let Some(Value::Summary { state: merged, .. }) = output.pop().and_then(|mut row| row.pop()) + else { + return Err("native summary merge did not return state".into()); + }; let Some(timestamp) = timestamp else { return Ok(MaintenanceValue::Summary { - state: Arc::from(merged), + state: merged, family, }); }; - result.insert(group, vec![(timestamp, Arc::from(merged))].into()); + result.insert(group, vec![(timestamp, merged)].into()); } Ok(MaintenanceValue::SummaryWindows { states: result, @@ -634,7 +837,7 @@ fn merge_inputs(inputs: &[Arc]) -> Result Result<[u8; 32], String> { +) -> Result<[u8; 32], MaintenanceError> { let generation = &inputs .first() .ok_or("immutable input cohort is empty")? @@ -649,31 +852,33 @@ fn frozen_cohort_lineage( } let mut ordered: Vec<_> = inputs.iter().collect(); ordered.sort_by(|left, right| { - (left.definition, left.sid, &left.group).cmp(&(right.definition, right.sid, &right.group)) + (&left.stored_output_reference, &left.group) + .cmp(&(&right.stored_output_reference, &right.group)) }); if ordered.windows(2).any(|pair| { - (pair[0].definition, pair[0].sid, &pair[0].group) - == (pair[1].definition, pair[1].sid, &pair[1].group) + (&pair[0].stored_output_reference, &pair[0].group) + == (&pair[1].stored_output_reference, &pair[1].group) }) { - return Err("immutable lineage repeats a physical population".into()); + return Err("immutable lineage repeats a stored-output population".into()); } let multiple = ordered.len() > 1; let mut lineage = Sha256::new(); - if multiple { - lineage.update(b"immutable-maintenance-input-v2"); - lineage.update((ordered.len() as u64).to_be_bytes()); - } else { - // Preserve the existing durable single-input receipt identity. - lineage.update(b"immutable-maintenance-input-v1"); - } + lineage.update(b"immutable-stored-output-input-v3"); + lineage.update((ordered.len() as u64).to_be_bytes()); for input in ordered { if &input.generation != generation || input.windows.is_empty() { return Err("immutable lineage has mixed generations or empty windows".into()); } - lineage.update(input.sid.to_be_bytes()); - let metadata = - serde_json::to_vec(&(&input.definition, &input.generation, &input.group, expected)) - .map_err(|error| error.to_string())?; + if input.stored_output_reference.stored_output_id != input.definition { + return Err("immutable input output differs from its definition".into()); + } + let metadata = serde_json::to_vec(&( + &input.stored_output_reference, + &input.generation, + &input.group, + expected, + )) + .map_err(|error| error.to_string())?; if multiple { lineage.update((metadata.len() as u64).to_be_bytes()); } @@ -706,7 +911,7 @@ fn prepare_frozen_maintenance_sink( planner_types::post_asap::ExecutableDag, MaterializationCommitKey, ), - String, + MaintenanceError, > { installed.validate()?; let target = match installed.binding.node(sink) { @@ -790,7 +995,7 @@ fn execute_prepared_frozen_sink( inputs: MaintenanceInputs<'_>, dag: &planner_types::post_asap::ExecutableDag, key: MaterializationCommitKey, -) -> Result<(SummaryState, Population), String> { +) -> Result<(SummaryState, Population), MaintenanceError> { let adapter = OperatorAdapter { binding: &installed.binding, inputs, @@ -800,9 +1005,10 @@ fn execute_prepared_frozen_sink( dag, &installed.binding, sink, - key, + key.clone(), &adapter, &CommitRegistry::default(), + maintenance_context(key.window_start_ms, key.window_end_ms, key.plan_version)?, ) .map_err(schedule_error)?; let group = match value.as_ref() { @@ -822,7 +1028,7 @@ fn evaluate_frozen_maintenance_sink( sink: PostAsapNodeId, input: &crate::storage_engines::sketch_db::index::FrozenExactWindows, output_window: (u64, u64), -) -> Result<(SummaryState, [u8; 32]), String> { +) -> Result<(SummaryState, [u8; 32]), MaintenanceError> { let (dag, key) = prepare_frozen_maintenance_sink( installed, configs, @@ -862,7 +1068,7 @@ pub fn execute_completed_maintenance( target_sid: u64, window: (u64, u64), group: &BTreeMap, -) -> Result { +) -> Result { let target = match installed.binding.node(sink) { Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("maintenance sink lacks an installed output identity".into()), @@ -904,7 +1110,7 @@ pub(crate) fn execute_completed_maintenance_cohort( target_sid: u64, window: (u64, u64), group: &BTreeMap, -) -> Result { +) -> Result { use asap_types::executable_plan::BackendNodeBinding; let target = match installed.binding.node(sink) { Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, @@ -1059,14 +1265,16 @@ pub(crate) fn execute_completed_maintenance_cohort( target.fingerprint(), ); output.catalog_generation = Some(Arc::clone(generation)); - store.publish_frozen_maintenance_output( - target_sid, - target_config, - &output, - state.as_ref(), - &cohort, - digest, - ) + store + .publish_frozen_maintenance_output( + target_sid, + target_config, + &output, + state.as_ref(), + &cohort, + digest, + ) + .map_err(Into::into) } /// Schedule a complete common population across all raw input definitions. @@ -1077,7 +1285,7 @@ fn execute_finite_source_cohort( plan: &asap_types::precompute_plan::PrecomputePlan, installed: &asap_types::executable_plan::InstalledPostAsapDag, sink: PostAsapNodeId, -) -> Result<(), String> { +) -> Result<(), MaintenanceError> { let generation = plan .summary_catalog .as_ref() @@ -1196,19 +1404,8 @@ fn execute_finite_source_cohort( config.population_key_encoding, &pairs, )?; - let kind = crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); let target_sid = - resolver.resolve_with_reactivation(&config.metric, &attrs, &kind, |sid| { - store.validate_routed_catalog_generation(Some(generation))?; - let activation = store.authorize_series_reactivation(sid, *target)?; - if activation - .as_deref() - .is_some_and(|actual| actual != generation) - { - return Err("finite maintenance generation changed".into()); - } - Ok(activation) - })?; + store.resolve_output_storage_handle(resolver, *target, &attrs, Some(generation))?; if existing .get(&target_sid) .and_then(|groups| groups.get(&output_group)) @@ -1241,7 +1438,7 @@ fn execute_finite_complete_populations( installed: &asap_types::executable_plan::InstalledPostAsapDag, sink: PostAsapNodeId, generation: &Arc, -) -> Result<(), String> { +) -> Result<(), MaintenanceError> { let target = match installed.binding.node(sink) { Some(BackendNodeBinding::Materialization { stored_output }) => *stored_output, _ => return Err("complete maintenance target is not bound".into()), @@ -1352,20 +1549,8 @@ fn execute_finite_complete_populations( config.population_key_encoding, &[], )?; - let kind = crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); - let target_sid = resolver - .resolve_with_reactivation(&config.metric, &attrs, &kind, |sid| { - store.validate_routed_catalog_generation(Some(generation))?; - let activation = store.authorize_series_reactivation(sid, target)?; - if activation - .as_deref() - .is_some_and(|actual| actual != generation.as_ref()) - { - return Err("complete maintenance generation changed".into()); - } - Ok(activation) - }) - .map_err(|error| error.to_string())?; + let target_sid = + store.resolve_output_storage_handle(resolver, target, &attrs, Some(generation))?; if store .completed_maintenance_coordinates(target, generation)? .get(&target_sid) @@ -1416,7 +1601,7 @@ pub(crate) fn execute_finite_maintenance( store: &crate::storage_engines::sketch_db::index::SketchStore, resolver: &crate::drivers::ingest::series_resolver::SeriesIdResolver, plan: &asap_types::precompute_plan::PrecomputePlan, -) -> Result<(), String> { +) -> Result<(), MaintenanceError> { let generation = plan .summary_catalog .as_ref() @@ -1507,25 +1692,11 @@ pub(crate) fn execute_finite_maintenance( config.population_key_encoding, &pairs, )?; - let kind = - crate::storage_engines::sketch_db::data::materialization_kind_for_config( - config, - ); - let target_sid = resolver.resolve_with_reactivation( - &config.metric, + let target_sid = store.resolve_output_storage_handle( + resolver, + *target, &attrs, - &kind, - |sid| { - store.validate_routed_catalog_generation(Some(generation))?; - let activation = store.authorize_series_reactivation(sid, *target)?; - if activation - .as_deref() - .is_some_and(|actual| actual != generation) - { - return Err("finite maintenance generation changed".into()); - } - Ok(activation) - }, + Some(generation), )?; for (start, _) in &windows { if (*start as i128 - config.pane_origin_ms.unwrap_or(0) as i128) @@ -1615,7 +1786,7 @@ struct CommitRegistry(Mutex); impl CommitRegistry { fn plan_snapshot( &self, - plans: &StreamingConfigHandle, + plans: &InstalledPrecomputePlanHandle, ) -> Result>, String> { let mut state = self.0.lock().map_err(|_| "commit registry poisoned")?; // Read the authoritative generation while holding the registry lock, @@ -1783,13 +1954,13 @@ impl IdempotentCommitSink for CommitRegistry { /// With no matching DAG, the source output is forwarded unchanged. pub struct MaintenanceDagSink { inner: Arc, - plans: StreamingConfigHandle, + plans: InstalledPrecomputePlanHandle, commits: CommitRegistry, batch_guard: Mutex<()>, } impl MaintenanceDagSink { - pub fn new(inner: Arc, plans: StreamingConfigHandle) -> Self { + pub fn new(inner: Arc, plans: InstalledPrecomputePlanHandle) -> Self { Self { inner, plans, @@ -1803,7 +1974,7 @@ impl MaintenanceDagSink { plan: &crate::storage_engines::types::RuntimePhysicalPlan, output: PrecomputedOutput, state: Box, - ) -> Result, String> { + ) -> Result, MaintenanceError> { let source_definition: asap_types::sds::StoredOutputId = output.policy_fp.into(); let source: SummaryState = Arc::from(state); let mut derived = Vec::new(); @@ -1852,6 +2023,8 @@ impl MaintenanceDagSink { }, configs: &plan.precompute_plan.materializations, }; + let mut selected_outputs = Vec::new(); + let mut horizons = Vec::new(); for sink_node in &installed.binding.precompute_sinks { // Derived summaries consume complete immutable windows at the // completion barrier, never additive worker fragments. @@ -1932,18 +2105,29 @@ impl MaintenanceDagSink { if self.commits.is_published(&key)? { continue; } - let value = execute_precompute_sink( - &dag, - &installed.binding, - *sink_node, - key.clone(), - &adapter, - &self.commits, - ) - .map_err(schedule_error)?; + selected_outputs.push((*sink_node, key)); + horizons.push(horizon_ms); + } + let values = execute_precompute_sinks( + &dag, + &installed.binding, + &selected_outputs, + &adapter, + &self.commits, + maintenance_context( + output.start_timestamp as i64, + output.end_timestamp as i64, + plan.plan_version(), + )?, + ) + .map_err(schedule_error)?; + for (((_, key), horizon_ms), value) in + selected_outputs.into_iter().zip(horizons).zip(values) + { + let target = key.stored_output; let mut target_output = output.clone(); target_output.policy_fp = target.into(); - target_output.series_id = None; + target_output.storage_handle = None; derived.push(( Some((key, horizon_ms)), target_output, @@ -1959,9 +2143,11 @@ impl MaintenanceDagSink { } } -fn schedule_error(error: ScheduleError) -> String { +fn schedule_error(error: ScheduleError) -> MaintenanceError { match error { - ScheduleError::Invalid(e) | ScheduleError::Operator(e) | ScheduleError::Sink(e) => e, + ScheduleError::Invalid(e) | ScheduleError::Sink(e) => e.into(), + ScheduleError::Operator(e) => e, + ScheduleError::Execution(e) => Box::new(e), } } @@ -2148,6 +2334,19 @@ pub(crate) fn affected_materializations( #[cfg(test)] mod tests { + fn test_context() -> asap_physical_operators::dag::RunContext { + use asap_physical_operators::dag::{Limits, RunContext, Scope}; + RunContext::new( + Scope::Ingestion { + window_start_ms: 0, + window_end_ms: 10_000, + revision: 1, + }, + Limits::default(), + ) + .unwrap() + } + use super::*; use asap_physical_operators::summary_kernels::SumAccumulator; use planner_types::post_asap::{ @@ -2186,7 +2385,10 @@ mod tests { let mut state = asap_physical_operators::summary_kernels::SumAccumulator::new(); state.update(value); FrozenExactWindows { - sid, + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + definition(id), + ), + storage_handle: sid, definition: definition(id), generation: Arc::new(asap_types::sds::CatalogGeneration { schema_version: 2, @@ -2205,6 +2407,13 @@ mod tests { }; let baseline = frozen_cohort_lineage(&[make(10, 1, 3.0), make(20, 2, 5.0)], &expected).unwrap(); + let mut relocated = make(20, 2, 5.0); + relocated.storage_handle = 999; + assert_eq!( + baseline, + frozen_cohort_lineage(&[make(10, 1, 3.0), relocated], &expected).unwrap(), + "local row relocation must not change stored-output lineage" + ); assert_eq!( baseline, frozen_cohort_lineage(&[make(20, 2, 5.0), make(10, 1, 3.0)], &expected).unwrap() @@ -2217,6 +2426,9 @@ mod tests { baseline, frozen_cohort_lineage(&[make(10, 1, 3.0), make(21, 2, 5.0)], &expected).unwrap() ); + let mut changed_output = make(20, 2, 5.0); + changed_output.stored_output_reference.stored_output_id.0 += 100; + assert!(frozen_cohort_lineage(&[make(10, 1, 3.0), changed_output], &expected).is_err()); let mut changed = make(20, 2, 5.0); changed.group.insert("instance".into(), "other".into()); assert_ne!( @@ -2241,6 +2453,42 @@ mod tests { .is_err()); } + // Ingestion adapters must execute native operators in the parent's scope. + #[test] + fn native_merge_uses_the_parent_budget_and_cancellation() { + let input = Arc::new(MaintenanceValue::summary(Arc::new( + SumAccumulator::with_sum(3.), + ))); + let context = test_context(); + let output = merge_inputs(&[Arc::clone(&input)], &context).unwrap(); + assert_eq!( + output + .state() + .unwrap() + .query_statistic(asap_types::Statistic::Sum, &None, &Default::default()) + .unwrap(), + 3. + ); + assert!(context.peak_bytes() > 0); + context.cancel(); + let error = merge_inputs(&[input], &context).err().unwrap(); + let error = error + .downcast_ref::() + .expect("native cancellation must retain its physical error type"); + fn cause( + error: &asap_physical_operators::dag::Error, + ) -> &asap_physical_operators::dag::Error { + match error { + asap_physical_operators::dag::Error::AtNode { source, .. } => cause(source), + error => error, + } + } + assert_eq!( + cause(error), + &asap_physical_operators::dag::Error::Cancelled + ); + } + fn node(id: u32) -> ExecutableDagNode { ExecutableDagNode { id: PostAsapNodeId(id), @@ -2279,7 +2527,10 @@ mod tests { fn frozen_adapter_resolves_each_materialized_frontier_without_aliasing() { use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType, SummaryField}; let make = |id| crate::storage_engines::sketch_db::index::FrozenExactWindows { - sid: id, + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + definition(id), + ), + storage_handle: id, definition: definition(id), generation: Arc::new(asap_types::sds::CatalogGeneration { schema_version: 2, @@ -2333,7 +2584,11 @@ mod tests { .unwrap() .is_none()); let merged = adapter - .execute(&node(3), &[Arc::new(first), Arc::new(second)]) + .execute( + &node(3), + &[Arc::new(first), Arc::new(second)], + test_context(), + ) .unwrap(); assert_eq!( merged @@ -2418,7 +2673,10 @@ mod tests { }; let frozen_inputs = [ crate::storage_engines::sketch_db::index::FrozenExactWindows { - sid: 1, + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + source_definition, + ), + storage_handle: 1, definition: source_definition, generation: Arc::new(asap_types::sds::CatalogGeneration { schema_version: 2, @@ -2452,7 +2710,7 @@ mod tests { ExactParams::Sum, )), }); - let row = adapter.execute(&read, &[source]).unwrap(); + let row = adapter.execute(&read, &[source], test_context()).unwrap(); let mut aggregate = node(3); aggregate.payload = ExecutableOperatorPayload::SummaryAgg { family: target_family, @@ -2464,7 +2722,9 @@ mod tests { reduction: Reduction::by(vec![]), grouping: GroupingStrategy::default(), }; - let result = adapter.execute(&aggregate, &[Arc::new(row)]).unwrap(); + let result = adapter + .execute(&aggregate, &[Arc::new(row)], test_context()) + .unwrap(); let mut kwargs = std::collections::HashMap::new(); kwargs.insert("quantile".into(), "0.5".into()); assert_eq!( @@ -2541,6 +2801,7 @@ mod tests { key, &scheduled_adapter, &CommitRegistry::default(), + test_context(), ) .unwrap(); assert_eq!( @@ -3109,7 +3370,7 @@ mod tests { .load_strict() .unwrap() .iter() - .all(|record| record.sid != 1)); + .all(|record| record.storage_handle != 1)); persistence.shutdown(); return; } @@ -3283,7 +3544,7 @@ mod tests { &BTreeMap::new(), ) .unwrap_err(); - assert!(stale.contains("stale producer"), "{stale}"); + assert!(stale.to_string().contains("stale producer"), "{stale}"); assert!(execute_finite_maintenance(&restored, &resolver, &plan).is_err()); assert_eq!(persistence.manifest.live_parts().len(), parts); assert!(persistence @@ -3292,7 +3553,7 @@ mod tests { .load_strict() .unwrap() .iter() - .all(|record| record.sid != 2)); + .all(|record| record.storage_handle != 2)); persistence.shutdown(); } @@ -3330,9 +3591,13 @@ mod tests { let left = rows(vec![(2_000, 7.0), (1_000, 5.0)]); let right = rows(vec![(1_000, 2.0), (2_000, 3.0)]); // Arrival order cannot exchange windows, and subtraction retains edge order. - let MaintenanceValue::Rows { values, .. } = - evaluate_aligned_binary(&operation, &operator, &[left.clone(), right.clone()]).unwrap() - else { + let MaintenanceValue::Rows { values, .. } = evaluate_aligned_binary( + &operation, + &operator, + &[left.clone(), right.clone()], + &test_context(), + ) + .unwrap() else { panic!("expected rows") }; assert_eq!(values[&BTreeMap::new()], vec![(1_000, 3.0), (2_000, 4.0)]); @@ -3359,6 +3624,7 @@ mod tests { &operation, &operator, &[grouped_left.clone(), grouped_right], + &test_context(), ) .unwrap() else { panic!("expected grouped rows") @@ -3373,7 +3639,8 @@ mod tests { &[ grouped_left, grouped(BTreeMap::from([(a, vec![(1_000, 2.0)])])) - ] + ], + &test_context() ) .is_err()); let binding = BackendExecutableBinding { @@ -3392,7 +3659,7 @@ mod tests { }; operation.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; assert!(frozen - .execute(&operation, &[left.clone(), right.clone()]) + .execute(&operation, &[left.clone(), right.clone()], test_context()) .is_ok()); let live = OperatorAdapter { binding: &binding, @@ -3403,14 +3670,14 @@ mod tests { configs: &[], }; assert!(live - .execute(&operation, &[left.clone(), right.clone()]) + .execute(&operation, &[left.clone(), right.clone()], test_context()) .is_err()); operation.payload = ExecutableOperatorPayload::Binary { operator: operator.clone(), }; operation.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; assert!(frozen - .execute(&operation, &[left.clone(), right.clone()]) + .execute(&operation, &[left.clone(), right.clone()], test_context()) .is_err()); operation.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; @@ -3427,19 +3694,27 @@ mod tests { }), Arc::new(MaintenanceValue::summary(sum(2.0))), ] { - assert!( - evaluate_aligned_binary(&operation, &operator, &[left.clone(), invalid]).is_err() - ); + assert!(evaluate_aligned_binary( + &operation, + &operator, + &[left.clone(), invalid], + &test_context() + ) + .is_err()); } operator.kind = BinaryOpKind::Arithmetic(ArithmeticOpKind::Div); assert!(evaluate_aligned_binary( &operation, &operator, - &[left.clone(), rows(vec![(1_000, 0.0), (2_000, 3.0)])] + &[left.clone(), rows(vec![(1_000, 0.0), (2_000, 3.0)])], + &test_context() ) .is_err()); operation.output_schema.fields[1].dtype = SummaryFamilyType::Plain(DataType::Int64); - assert!(evaluate_aligned_binary(&operation, &operator, &[left, right]).is_err()); + assert!( + evaluate_aligned_binary(&operation, &operator, &[left, right], &test_context()) + .is_err() + ); } #[test] @@ -3460,7 +3735,7 @@ mod tests { )]), family, }); - assert!(merge_inputs(&[inputs.clone(), different_group]).is_err()); + assert!(merge_inputs(&[inputs.clone(), different_group], &test_context()).is_err()); let mut read = node(2); read.output_schema.fields = vec![SummaryField { name: "value".into(), @@ -3468,15 +3743,16 @@ mod tests { nullable: false, }]; let MaintenanceValue::Rows { values, .. } = - finalize_exact(&read, &[inputs.clone()]).unwrap() + finalize_exact(&read, &[inputs.clone()], &test_context()).unwrap() else { panic!("expected finalized rows") }; assert_eq!(values[&BTreeMap::new()], vec![(1_000, 2.0), (2_000, 7.0)]); // Merge is a semantic DAG operation, not an implicit batch optimization. // Finalizing after it emits exactly one value instead of two updates. - let merged = Arc::new(merge_inputs(&[inputs]).unwrap()); - let MaintenanceValue::Rows { values, .. } = finalize_exact(&read, &[merged]).unwrap() + let merged = Arc::new(merge_inputs(&[inputs], &test_context()).unwrap()); + let MaintenanceValue::Rows { values, .. } = + finalize_exact(&read, &[merged], &test_context()).unwrap() else { panic!("expected finalized row") }; @@ -3491,7 +3767,7 @@ mod tests { )), }); assert!( - matches!(finalize_exact(&read, &[integer_state]), Err(error) if error.contains("Float64")) + matches!(finalize_exact(&read, &[integer_state], &test_context()), Err(error) if error.to_string().contains("Float64")) ); } @@ -3518,7 +3794,7 @@ mod tests { ]; read.output_schema.time_index = Some(0); let MaintenanceValue::Rows { values, name, .. } = - finalize_exact(&read, &[Arc::clone(&input)]).unwrap() + finalize_exact(&read, &[Arc::clone(&input)], &test_context()).unwrap() else { panic!("expected typed rows") }; @@ -3546,7 +3822,7 @@ mod tests { copy.output_schema.fields[1].name = "ts".into(); malformed.push(copy); for malformed in malformed { - assert!(finalize_exact(&malformed, &[Arc::clone(&input)]).is_err()); + assert!(finalize_exact(&malformed, &[Arc::clone(&input)], &test_context()).is_err()); } let untimed = Arc::new(MaintenanceValue::Summary { state: sum(10.0), @@ -3555,7 +3831,7 @@ mod tests { ExactParams::Sum, )), }); - assert!(finalize_exact(&read, &[untimed]).is_err()); + assert!(finalize_exact(&read, &[untimed], &test_context()).is_err()); } #[test] @@ -3586,8 +3862,14 @@ mod tests { reduction: Reduction::by(vec![]), grouping: GroupingStrategy::default(), }; - let error = adapter.execute(&aggregate, &[Arc::new(MaintenanceValue::summary(sum(7.0)))]); - assert!(matches!(error, Err(reason) if reason.contains("typed update evaluator"))); + let error = adapter.execute( + &aggregate, + &[Arc::new(MaintenanceValue::summary(sum(7.0)))], + test_context(), + ); + assert!( + matches!(error, Err(reason) if reason.to_string().contains("typed update evaluator")) + ); } #[test] @@ -3647,8 +3929,10 @@ mod tests { name: "value".into(), timestamped: true, }; - assert!(matches!(adapter.execute(&aggregate, &[Arc::new(rows)]), - Err(error) if error.contains("one explicitly reduced output population"))); + assert!( + matches!(adapter.execute(&aggregate, &[Arc::new(rows)], test_context()), + Err(error) if error.to_string().contains("one explicitly reduced output population")) + ); } #[test] @@ -3713,8 +3997,10 @@ mod tests { name: "value".into(), timestamped: true, }; - assert!(matches!(adapter.execute(&aggregate, &[Arc::new(rows)]), - Err(error) if error.contains("positive representable domain"))); + assert!( + matches!(adapter.execute(&aggregate, &[Arc::new(rows)], test_context()), + Err(error) if error.to_string().contains("positive representable domain")) + ); } } @@ -3818,7 +4104,7 @@ mod tests { #[test] fn downstream_failure_does_not_acknowledge_maintenance_publication() { use crate::storage_engines::types::{ - ActivePhysicalPlanHandle, RuntimePhysicalPlan, StreamingConfig, + ActivePhysicalPlanHandle, InstalledPrecomputePlan, RuntimePhysicalPlan, }; use asap_types::executable_plan::{InstalledPostAsapDag, OwnedPostAsapDag}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -3914,7 +4200,7 @@ mod tests { summary_catalog: Some(Arc::new(bundle.summary_catalog)), precompute_plan: bundle.precompute_plan, transmission_plan: bundle.transmission_plan, - streaming_config: Arc::new(StreamingConfig::new(Default::default())), + installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(Default::default())), query_plan: Arc::new(bundle.query_plan), storage_routing: Arc::new(Default::default()), }; @@ -3929,9 +4215,9 @@ mod tests { }); let sink = MaintenanceDagSink::new( downstream.clone(), - StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new( - active.clone(), - )), + InstalledPrecomputePlanHandle::from_active_physical_plan( + ActivePhysicalPlanHandle::new(active.clone()), + ), ); let batch = || { (0..count) @@ -4057,6 +4343,7 @@ mod tests { key.clone(), &adapter, &commits, + test_context(), ) .unwrap(); assert_eq!(result.state().unwrap().aux_stats().sum, Some(4.0)); @@ -4132,10 +4419,208 @@ mod tests { PostAsapNodeId(1), key.clone(), &adapter, - &commits + &commits, + test_context() ), Err(ScheduleError::Operator(_)) )); assert!(commits.get(&key).unwrap().is_none()); } } + +/// Captured local input provides fixed population membership for this revision. +/// All compatible sinks share one native execution cache for each output window. +pub(crate) fn execute_revision_outputs( + plan: &crate::storage_engines::types::RuntimePhysicalPlan, + raw: &[crate::storage_engines::sketch_db::index::FrozenExactWindows], + revision: u64, + limit: usize, + publish: &mut crate::precompute_engine::revisions::Publisher<'_>, +) -> Result<(), MaintenanceError> { + use crate::precompute_engine::revisions::{encode_state, RevisionRecord}; + let generation = plan + .precompute_plan + .summary_catalog + .as_ref() + .ok_or("revision generation missing")?; + for installed in plan.precompute_plan.executable_dags.values() { + let mut windows: BTreeMap<(u64, u64), Vec<(PostAsapNodeId, MaterializationCommitKey)>> = + BTreeMap::new(); + let mut result: BTreeMap> = + BTreeMap::new(); + for sink in &installed.binding.precompute_sinks { + let Some(BackendNodeBinding::Materialization { + stored_output: target, + }) = installed.binding.node(*sink) + else { + continue; + }; + let config = plan + .precompute_plan + .materializations + .iter() + .find(|c| c.policy_fingerprint() == target.fingerprint()) + .ok_or("revision target configuration missing")?; + let Some(derived) = &config.derived_input else { + continue; + }; + result.entry(*target).or_default(); + let inputs: Vec<_> = raw + .iter() + .filter(|r| derived.inputs.contains(&r.definition)) + .collect(); + let width = config.stored_window_ms(); + if width == 0 { + return Err("revision output has zero window extent".into()); + } + let possible: BTreeSet<_> = inputs + .iter() + .flat_map(|i| i.windows.keys()) + .filter_map(|(start, _)| { + ((*start as i128 - config.pane_origin_ms.unwrap_or(0) as i128) + .rem_euclid(width as i128) + == 0) + .then(|| start.checked_add(width).map(|end| (*start, end))) + .flatten() + }) + .collect(); + for window in possible { + let selected: Vec<_> = inputs + .iter() + .map( + |input| crate::storage_engines::sketch_db::index::FrozenExactWindows { + stored_output_reference: input.stored_output_reference.clone(), + storage_handle: input.storage_handle, + definition: input.definition, + generation: Arc::clone(&input.generation), + group: input.group.clone(), + windows: input + .windows + .iter() + .filter(|((s, e), _)| *s >= window.0 && *e <= window.1) + .map(|(w, s)| (*w, Arc::clone(s))) + .collect(), + singleton_population_complete: true, + }, + ) + .collect(); + // Every contributing population must have contiguous pane coverage. + if selected.is_empty() + || selected.iter().any(|i| { + let mut cursor = window.0; + for (start, end) in i.windows.keys() { + if *start != cursor { + return true; + } + cursor = *end; + } + cursor != window.1 + }) + { + continue; + } + let (_, mut key) = prepare_frozen_maintenance_sink( + installed, + &plan.precompute_plan.materializations, + *sink, + &selected, + window, + )?; + // The captured input revision, rather than a sink-specific digest, + // identifies the common evaluation frontier for shared producers. + key.input_lineage = revision.to_be_bytes().to_vec(); + windows.entry(window).or_default().push((*sink, key)); + } + } + let dag = installed.document.decode()?; + for (window, sinks) in windows { + let inputs: Vec<_> = raw + .iter() + .filter_map(|input| { + let windows: BTreeMap<_, _> = input + .windows + .iter() + .filter(|((s, e), _)| *s >= window.0 && *e <= window.1) + .map(|(w, s)| (*w, Arc::clone(s))) + .collect(); + (!windows.is_empty()).then(|| { + crate::storage_engines::sketch_db::index::FrozenExactWindows { + stored_output_reference: input.stored_output_reference.clone(), + storage_handle: input.storage_handle, + definition: input.definition, + generation: Arc::clone(&input.generation), + group: input.group.clone(), + windows, + singleton_population_complete: true, + } + }) + }) + .collect(); + let adapter = OperatorAdapter { + binding: &installed.binding, + inputs: MaintenanceInputs::Captured(&inputs), + configs: &plan.precompute_plan.materializations, + }; + let context = RunContext::new( + asap_physical_operators::dag::Scope::Ingestion { + window_start_ms: window.0 as i64, + window_end_ms: window.1 as i64, + revision, + }, + asap_physical_operators::dag::Limits { + max_bytes: limit, + ..Default::default() + }, + )?; + let values = execute_precompute_sinks( + &dag, + &installed.binding, + &sinks, + &adapter, + &CommitRegistry::default(), + context, + ) + .map_err(schedule_error)?; + for ((_, key), value) in sinks.iter().zip(values) { + let group = match value.as_ref() { + MaintenanceValue::SummaryWindows { states, .. } if states.len() == 1 => { + states.keys().next().unwrap().clone() + } + MaintenanceValue::Summary { .. } => BTreeMap::new(), + _ => { + return Err( + "revision sink must explicitly reduce its output population".into() + ) + } + }; + let config = plan + .precompute_plan + .materializations + .iter() + .find(|c| c.policy_fingerprint() == key.stored_output.fingerprint()) + .ok_or("revision output configuration missing")?; + result + .get_mut(&key.stored_output) + .unwrap() + .push(RevisionRecord { + reference: plan + .installed_precompute_plan + .stored_output_reference(key.stored_output) + .ok_or("revision output binding missing")?, + group, + start_ms: window.0, + end_ms: window.1, + payload: encode_state( + Arc::clone(value.state()?), + config.accumulator_spec()?.family, + )?, + }); + } + } + for (output, records) in result { + publish(output, records)?; + } + } + let _ = generation; + Ok(()) +} diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index 7c8176b50..1aaa257e4 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -18,3 +18,7 @@ pub mod worker; pub use engine::{PrecomputeEngine, PrecomputeWorkerDiagnostics}; pub use ingest_handler::IngestState; + +pub mod partitioning; + +pub mod revisions; diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index d32388b78..42f2562c1 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -1,7 +1,7 @@ use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::ingest_handler::IngestObservability; use crate::storage_engines::sketch_db::index::SketchStore; -use crate::storage_engines::types::hot_reload_config::StreamingConfigHandle; +use crate::storage_engines::types::hot_reload_config::InstalledPrecomputePlanHandle; use crate::storage_engines::types::{AggregateCore, PrecomputedOutput}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -45,20 +45,14 @@ fn consume_in_order(items: Vec, mut persist: impl FnMut(&T) -> bool) -> us failed } -/// Write completed precompute windows to the sid-keyed sketch store. -/// -/// Read one streaming-config snapshot per batch and resolve each policy through -/// its fingerprint. Consume accumulators in order so catch-up batches release -/// each pane as soon as it is serialized. +/// Publish completed windows using their selected output reference and complete +/// plan/population/window address. Local row handles are checked against this +/// address and cannot redirect a queued output to another stored summary. pub struct SketchStoreSink { summary_store: Arc, - hot_reload: StreamingConfigHandle, - /// Single shared resolver across the ingest + precompute paths. Under - /// the registry-allocated sid model (PR-1..3), this is the canonical - /// mint authority — precompute sids share the same `next_sid` counter - /// as OTel-sketch sids, so the two paths can never collide on identity - /// even when the same `(metric, attrs)` carries both a sketch and an - /// exact precompute. + hot_reload: InstalledPrecomputePlanHandle, + /// Durable local-handle allocation keyed by the complete output population + /// identity. Shared with ingest and backfill so they address the same rows. series_resolver: Arc, /// CQ-6 — optional handle to the shared `IngestObservability` so /// policy-miss drops land in the same counter the OTLP ingest path @@ -73,7 +67,7 @@ pub struct SketchStoreSink { impl SketchStoreSink { pub fn new( summary_store: Arc, - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, series_resolver: Arc, ) -> Self { Self { @@ -145,47 +139,41 @@ impl SketchStoreSink { } }; let agg_cfg = &agg_cfg; - let resolver = self.series_resolver.clone(); + let Some(reference) = cfg.stored_output_reference(output.policy_fp.into()) else { + return false; + }; + let Some(start_ms) = i64::try_from(output.start_timestamp).ok() else { + return false; + }; + let Some(end_ms) = i64::try_from(output.end_timestamp).ok() else { + return false; + }; + let population = output.population_labels.clone().unwrap_or_else(|| { + agg_cfg + .grouping_labels + .iter() + .cloned() + .zip(output.key.clone().unwrap_or_default().labels) + .collect() + }); + let (plan_id, plan_version) = output + .catalog_generation + .as_deref() + .map(|generation| (generation.plan_id, generation.plan_version)) + .unwrap_or((0, 0)); + let address = asap_types::sds::StoredSummaryKey { + plan_id, + plan_version, + stored_output_id: reference.stored_output_id, + population, + window: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, + }; let persist = |writer: &crate::storage_engines::sketch_db::index::SummaryPublicationWriter<'_>| { - if let Some(sid) = output.series_id { - return writer - .ingest_precompute_with_series_id(sid, agg_cfg, output, accumulator) - .inspect(|_| { - crate::precompute_engine::metrics::record_materialized_outputs(1) - }); - } - self.summary_store - .validate_routed_catalog_generation(output.catalog_generation.as_deref()) - .ok()?; writer - .ingest_precompute_for_agg_config( - |metric, fp, ak| { - resolver - .resolve_with_reactivation(metric, fp, ak, |sid| { - self.summary_store.validate_routed_catalog_generation( - output.catalog_generation.as_deref(), - )?; - let activation = - self.summary_store.authorize_series_reactivation( - sid, - output.policy_fp.into(), - )?; - if let Some(generation) = &activation { - if output.catalog_generation.as_deref() - != Some(generation.as_ref()) - { - return Err( - "unbound or stale output cannot reactivate a series" - .into(), - ); - } - } - Ok(activation) - }) - .map_err(|error| warn!(%error, "series reactivation rejected")) - .ok() - }, + .write_stored_summary( + &address, + &self.series_resolver, agg_cfg, output, accumulator, @@ -336,7 +324,7 @@ impl OutputSink for NoopOutputSink { mod tests { use super::*; use crate::storage_engines::sketch_db::index::{AggKind, SeriesLookup}; - use crate::storage_engines::types::{KeyByLabelValues, StreamingConfig}; + use crate::storage_engines::types::{InstalledPrecomputePlan, KeyByLabelValues}; use asap_physical_operators::summary_kernels::{DDSketchAccumulator, SumAccumulator}; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; @@ -427,8 +415,8 @@ mod tests { let agg_id = cfg.policy_fp_u64(); let mut configs = HashMap::new(); configs.insert(agg_id, cfg); - let streaming = StreamingConfig::new(configs); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(configs); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( @@ -453,7 +441,7 @@ mod tests { .list_by_status(crate::storage_engines::sketch_db::lifecycle::AggStatus::Active); assert_eq!(instances.len(), 1); let meta = instances[0].clone(); - let sid = meta.sid; + let sid = meta.storage_handle; assert_eq!(summary_store.classify(sid), SeriesLookup::Hit); assert!( matches!( @@ -496,7 +484,10 @@ mod tests { Arc::new(SeriesIdResolver::open(temporary.path().join("resolver.wal")).unwrap()); let sink = SketchStoreSink::new( store.clone(), - StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(fingerprint.0, cfg)]))), + InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new(HashMap::from([( + fingerprint.0, + cfg, + )]))), resolver, ); let original_generation = Arc::new(catalog.reference().unwrap()); @@ -513,7 +504,7 @@ mod tests { .emit_batch(vec![(output(), Box::new(SumAccumulator::with_sum(11.0)))]) .is_err()); let mut stale_output = output(); - stale_output.series_id = Some(old_sid); + stale_output.storage_handle = Some(old_sid); stale_output.catalog_generation = Some(Arc::new(catalog.reference().unwrap())); let mut next = catalog; next.plan_version += 1; @@ -544,7 +535,7 @@ mod tests { .emit_batch(vec![(output(), Box::new(SumAccumulator::with_sum(101.0)))]) .is_err()); let mut stale_routed_output = output(); - stale_routed_output.series_id = Some(new_sid); + stale_routed_output.storage_handle = Some(new_sid); assert!(sink .emit_batch(vec![( stale_routed_output, @@ -552,7 +543,7 @@ mod tests { )]) .is_err()); let mut missing_generation = PrecomputedOutput::new(1000, 2000, None, fingerprint); - missing_generation.series_id = Some(new_sid); + missing_generation.storage_handle = Some(new_sid); assert!(sink .emit_batch(vec![( missing_generation, @@ -575,8 +566,9 @@ mod tests { cfg.parameters .insert("alpha".into(), serde_json::json!(0.01)); let policy_fp = cfg.policy_fp_u64(); - let hot_reload = - StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(policy_fp, cfg)]))); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new( + HashMap::from([(policy_fp, cfg)]), + )); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( summary_store.clone(), @@ -604,9 +596,14 @@ mod tests { .. } )); - assert_eq!(summary_store.query_range(meta.sid, 1_000, 2_000).len(), 1); + assert_eq!( + summary_store + .query_range(meta.storage_handle, 1_000, 2_000) + .len(), + 1 + ); assert!(summary_store - .query_exact_agg_range(meta.sid, 1_000, 2_000) + .query_exact_agg_range(meta.storage_handle, 1_000, 2_000) .is_empty()); } @@ -614,8 +611,8 @@ mod tests { fn sketch_index_sink_reports_unknown_policy_as_failure() { // Streaming config does NOT contain agg_id=99 — the sink // reports a recoverable error rather than acknowledging a lost write. - let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(HashMap::new()); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( summary_store.clone(), @@ -635,8 +632,8 @@ mod tests { /// handle wired in, the `dropped_policy_miss` counter must tick. #[test] fn sink_increments_policy_miss_counter_on_registry_miss() { - let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(HashMap::new()); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); let obs = Arc::new(IngestObservability::new()); let sink = SketchStoreSink::new( diff --git a/data_plane/src/precompute_engine/partitioning.rs b/data_plane/src/precompute_engine/partitioning.rs new file mode 100644 index 000000000..09ef174fd --- /dev/null +++ b/data_plane/src/precompute_engine/partitioning.rs @@ -0,0 +1,115 @@ +//! Data ownership for an entire installed precompute graph. +use std::collections::{BTreeMap, BTreeSet}; + +use asap_types::precompute_plan::PrecomputePlan; +use xxhash_rust::xxh64::xxh64; + +/// A conservative locality proof. Raw producer paths are validated separately +/// at installation. Derived graphs use one owner until their complete reduction +/// paths can establish a finer partition without a cross-worker exchange. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum DagPartitioning { + #[default] + SingleWorker, + Labels(Vec), + Population, +} + +impl DagPartitioning { + pub fn from_plan(plan: &PrecomputePlan) -> Self { + if plan + .materializations + .iter() + .any(|output| output.derived_input.is_some()) + { + return Self::SingleWorker; + } + let mut common: Option> = None; + for output in &plan.materializations { + let labels = output + .grouping_labels + .iter() + .cloned() + .collect::>(); + common = Some(match common { + None => labels, + Some(previous) => previous.intersection(&labels).cloned().collect(), + }); + } + match common { + Some(labels) if !labels.is_empty() => Self::Labels(labels.into_iter().collect()), + _ if !plan.materializations.is_empty() + && plan.materializations.iter().all(|output| { + output.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) + }) => + { + Self::Population + } + _ => Self::SingleWorker, + } + } + + pub fn owner( + &self, + population: &BTreeMap, + workers: usize, + ) -> Result { + if workers == 0 { + return Err("precompute requires at least one worker".into()); + } + match self { + Self::SingleWorker => Ok(0), + Self::Population => { + let key = super::group_key::GroupKey::new( + population.iter().map(|(k, v)| (k.as_str(), v.as_str())), + ); + Ok(xxh64(key.canonical_bytes(), 0) as usize % workers) + } + Self::Labels(names) => { + let pairs = names + .iter() + .map(|name| { + population + .get(name) + .map(|value| (name.as_str(), value.as_str())) + .ok_or_else(|| { + format!("DAG partition label {name} is absent from population") + }) + }) + .collect::, _>>()?; + let key = super::group_key::GroupKey::new(pairs); + Ok(xxh64(key.canonical_bytes(), 0) as usize % workers) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Upstream series state and downstream grouped state have one owner. + #[test] + fn service_partition_preserves_series_and_reduction_locality() { + let rule = DagPartitioning::Labels(vec!["service".into()]); + let group = BTreeMap::from([("service".into(), "api".into())]); + let expected = rule.owner(&group, 4).unwrap(); + for instance in ["a", "b", "c"] { + let mut series = group.clone(); + series.insert("instance".into(), instance.into()); + assert_eq!(rule.owner(&series, 4).unwrap(), expected); + } + assert!(rule.owner(&BTreeMap::new(), 4).is_err()); + assert!(rule.owner(&group, 0).is_err()); + let owners = (0..64) + .map(|i| { + rule.owner( + &BTreeMap::from([("service".into(), format!("service-{i}"))]), + 4, + ) + .unwrap() + }) + .collect::>(); + assert_eq!(owners.len(), 4); + } +} diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs index 95c0fe0cb..fac1323a1 100644 --- a/data_plane/src/precompute_engine/raw_dag.rs +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -223,13 +223,13 @@ impl RawDagProgram { self.updater().map(|_| ()) } - pub fn apply( + /// Validate admission with the same weight semantics used during execution, + /// without mutating an accumulator or accepting part of a request. + pub fn validate_sample( &self, - updater: &mut dyn AccumulatorUpdater, - series: &str, + updater: &dyn AccumulatorUpdater, value: f64, - timestamp: i64, - ) -> Result<(), String> { + ) -> Result { let weight = match &self.input.weight { SummaryInputExpr::Constant(c) => *c, // The worker retains one previous value per series across pane rotation. @@ -240,6 +240,17 @@ impl RawDagProgram { && !updater.is_keyed(); let weight = if scalar_frequency { value } else { weight }; updater.validate_single_input(weight)?; + Ok(weight) + } + + pub fn apply( + &self, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + ) -> Result<(), String> { + let weight = self.validate_sample(updater, value)?; if updater.is_keyed() { let labels = super::worker::parse_labels_from_series_key(series); fn eval( diff --git a/data_plane/src/precompute_engine/revisions.rs b/data_plane/src/precompute_engine/revisions.rs new file mode 100644 index 000000000..fc9126ab0 --- /dev/null +++ b/data_plane/src/precompute_engine/revisions.rs @@ -0,0 +1,1010 @@ +//! Bounded local-input snapshots and independently published stored results. +//! The checkpoint is runtime state for one installed generation, not a new plan. +use asap_types::sds::{CatalogGeneration, StoredOutputId, StoredOutputReference}; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs::{self, File, OpenOptions}, + io::Write, + path::PathBuf, + sync::{Arc, Mutex, RwLock}, +}; + +pub type Publisher<'a> = + dyn FnMut(StoredOutputId, Vec) -> Result<(), RevisionError> + 'a; + +#[derive(Debug, thiserror::Error)] +#[error("no compatible input snapshot satisfies query freshness")] +pub struct SnapshotUnavailable; + +#[derive(Debug, thiserror::Error)] +#[error("revision admission rejected: {0}")] +pub struct AdmissionRejected(pub &'static str); + +pub type RevisionError = Box; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RevisionPolicy { + pub correction_horizon_ms: u64, + pub max_query_staleness_ms: u64, + pub max_checkpoint_bytes: usize, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct InputSample { + pub metric: String, + pub labels: BTreeMap, + pub series: String, + pub timestamp_ms: i64, + pub first_revision: u64, + pub value: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RevisionRecord { + pub reference: StoredOutputReference, + pub group: BTreeMap, + pub start_ms: u64, + pub end_ms: u64, + pub payload: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Snapshot { + revision: u64, + captured_at_ms: u64, + outputs: BTreeMap>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Checkpoint { + schema_version: u32, + generation: CatalogGeneration, + policy: RevisionPolicy, + inputs: Vec, + snapshots: Vec, +} + +/// Owned records pin the selected snapshot independently of later commits/GC. +#[derive(Debug, Clone)] +pub struct PinnedRevision { + pub revision: u64, + pub captured_at_ms: u64, + pub records: Vec, + pub inputs: Vec, +} + +pub struct RevisionStore { + path: PathBuf, + _writer: File, + execution: Mutex<()>, + state: RwLock>, + poisoned: std::sync::atomic::AtomicBool, +} + +impl RevisionStore { + pub fn open( + path: PathBuf, + generation: CatalogGeneration, + policy: RevisionPolicy, + ) -> Result { + if policy.correction_horizon_ms == 0 + || policy.max_query_staleness_ms == 0 + || policy.max_checkpoint_bytes == 0 + { + return Err( + "revision policy requires positive bounded horizons and a byte budget".into(), + ); + } + fs::create_dir_all( + path.parent() + .ok_or("revision checkpoint needs a parent directory")?, + )?; + let writer = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(path.with_extension("lock"))?; + writer.try_lock_exclusive()?; + if fs::metadata(&path) + .is_ok_and(|metadata| metadata.len() > policy.max_checkpoint_bytes as u64) + { + return Err("revision checkpoint exceeds configured byte budget".into()); + } + let document = match fs::read(&path) { + Ok(bytes) => { + if bytes.len() > policy.max_checkpoint_bytes { + return Err("revision checkpoint exceeds configured byte budget".into()); + } + let document: Checkpoint = serde_json::from_slice(&bytes)?; + if document.schema_version != 1 + || document.generation != generation + || document.policy != policy + { + return Err( + "revision checkpoint differs from installed generation or policy".into(), + ); + } + let mut input_keys = BTreeSet::new(); + for sample in &document.inputs { + if sample.timestamp_ms < 0 + || sample.first_revision == 0 + || document + .snapshots + .last() + .is_none_or(|s| sample.first_revision > s.revision) + || sample.value.is_some_and(|v| !v.is_finite()) + || !input_keys.insert((&sample.series, sample.timestamp_ms)) + { + return Err("revision checkpoint contains invalid or repeated input".into()); + } + } + let mut previous = (0, 0); + for snapshot in &document.snapshots { + if snapshot.revision <= previous.0 || snapshot.captured_at_ms < previous.1 { + return Err("revision checkpoint has non-monotonic snapshots".into()); + } + for (output, records) in &snapshot.outputs { + validate_records(*output, records)?; + } + previous = (snapshot.revision, snapshot.captured_at_ms); + } + document + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Checkpoint { + schema_version: 1, + generation, + policy, + inputs: Vec::new(), + snapshots: Vec::new(), + }, + Err(error) => return Err(error.into()), + }; + Ok(Self { + path, + _writer: writer, + execution: Mutex::new(()), + state: RwLock::new(Arc::new(document)), + poisoned: false.into(), + }) + } + + fn current(&self) -> Result, RevisionError> { + if self.poisoned.load(std::sync::atomic::Ordering::SeqCst) { + return Err("revision persistence is uncertain; reopen before continuing".into()); + } + Ok(self + .state + .read() + .map_err(|_| "revision state poisoned")? + .clone()) + } + + fn persist(&self, next: Checkpoint) -> Result<(), RevisionError> { + let bytes = serde_json::to_vec(&next)?; + if bytes.len() > next.policy.max_checkpoint_bytes { + return Err(Box::new(asap_physical_operators::dag::Error::MemoryLimit)); + } + let result = (|| -> std::io::Result<()> { + let temporary = self.path.with_extension("tmp"); + let mut file = File::create(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + fs::rename(temporary, &self.path)?; + File::open(self.path.parent().unwrap())?.sync_all() + })(); + if let Err(error) = result { + self.poisoned + .store(true, std::sync::atomic::Ordering::SeqCst); + return Err(error.into()); + } + *self.state.write().map_err(|_| "revision state poisoned")? = Arc::new(next); + Ok(()) + } + + /// Admission and execution are serialized; readers only hold the short state + /// lock. Each output commit is durable independently, allowing old snapshots + /// to remain readable if a later output fails. + #[allow(clippy::too_many_arguments)] + pub fn apply( + &self, + generation: &CatalogGeneration, + samples: Vec, + now_ms: u64, + retain_input_ms: u64, + outputs: &BTreeSet, + validate: impl FnOnce(&[InputSample], u64) -> Result<(), RevisionError>, + execute: impl FnOnce( + &[InputSample], + u64, + u64, + bool, + &mut Publisher<'_>, + ) -> Result<(), RevisionError>, + ) -> Result { + let _execution = self + .execution + .lock() + .map_err(|_| "revision execution poisoned")?; + let current = self.current()?; + if ¤t.generation != generation { + return Err("revision input belongs to a different installed generation".into()); + } + if current + .snapshots + .last() + .is_some_and(|s| now_ms < s.captured_at_ms) + { + return Err("revision clock regressed".into()); + } + validate(&samples, current.policy.correction_horizon_ms)?; + if retain_input_ms < current.policy.correction_horizon_ms { + return Err("input retention is shorter than correction horizon".into()); + } + let mut inputs: BTreeMap<_, _> = current + .inputs + .iter() + .cloned() + .map(|s| ((s.series.clone(), s.timestamp_ms), s)) + .collect(); + let mut changed = false; + let next_revision = current.snapshots.last().map_or(Ok(1), |s| { + s.revision.checked_add(1).ok_or("input revision exhausted") + })?; + for mut sample in samples { + if sample.value.is_some_and(|v| !v.is_finite()) { + return Err("revision input must be finite or an explicit stale marker".into()); + } + let key = (sample.series.clone(), sample.timestamp_ms); + sample.first_revision = inputs + .get(&key) + .map_or(next_revision, |old| old.first_revision); + match inputs.get(&key) { + Some(old) if old != &sample => { + return Err(AdmissionRejected("conflicting sample in revision input").into()) + } + Some(_) => {} + None => { + inputs.insert(key, sample); + changed = true; + } + } + } + let pending = current + .snapshots + .last() + .is_some_and(|s| outputs.iter().any(|o| !s.outputs.contains_key(&o.0))); + if changed && pending { + return Err( + "prior input revision is pending; retry its execution before new admission".into(), + ); + } + let revision = if changed + || current.snapshots.is_empty() + || (!pending + && current + .snapshots + .last() + .is_some_and(|s| now_ms > s.captured_at_ms)) + { + let revision = current.snapshots.last().map_or(Ok(1), |s| { + s.revision.checked_add(1).ok_or("input revision exhausted") + })?; + let floor = now_ms.saturating_sub(retain_input_ms); + let mut next = (*current).clone(); + next.inputs = inputs + .into_values() + .filter(|s| s.timestamp_ms >= 0 && s.timestamp_ms as u64 >= floor) + .collect(); + let cutoff = now_ms.saturating_sub(current.policy.max_query_staleness_ms); + next.snapshots.retain(|s| s.captured_at_ms >= cutoff); + next.snapshots.push(Snapshot { + revision, + captured_at_ms: now_ms, + outputs: BTreeMap::new(), + }); + self.persist(next)?; + revision + } else if !pending { + return Ok(current.snapshots.last().unwrap().revision); + } else { + current.snapshots.last().unwrap().revision + }; + let snapshot = self.current()?; + let mut publish = |output: StoredOutputId, records: Vec| { + if !outputs.contains(&output) { + return Err("revision publication references an uninstalled output".into()); + } + validate_records(output.0, &records)?; + let mut next = (*self.current()?).clone(); + let frame = next + .snapshots + .last_mut() + .ok_or("input revision disappeared")?; + if frame.revision != revision { + return Err("input revision changed during execution".into()); + } + // Reuse committed randomized sketches on recovery; do not recompute + // their identity from another random realization of the same input. + if frame.outputs.contains_key(&output.0) { + return Ok(()); + } + frame.outputs.insert(output.0, records); + self.persist(next) + }; + execute( + &snapshot.inputs, + revision, + snapshot.snapshots.last().unwrap().captured_at_ms, + changed, + &mut publish, + )?; + if outputs.iter().any(|o| { + !self + .current() + .map(|s| s.snapshots.last().unwrap().outputs.contains_key(&o.0)) + .unwrap_or(false) + }) { + return Err("revision execution did not publish every selected output".into()); + } + Ok(revision) + } + + pub(crate) fn committed_outputs( + &self, + revision: u64, + ) -> Result>, RevisionError> { + self.current()? + .snapshots + .iter() + .find(|s| s.revision == revision) + .map(|s| s.outputs.clone()) + .ok_or_else(|| "captured revision disappeared".into()) + } + + #[cfg(test)] + pub fn pin( + &self, + required: &BTreeSet, + now_ms: u64, + ) -> Result { + self.pin_matching(required, now_ms, |_| true) + } + + fn pin_matching( + &self, + required: &BTreeSet, + now_ms: u64, + eligible: impl Fn(&BTreeMap>) -> bool, + ) -> Result { + let state = self.current()?; + let snapshot = state + .snapshots + .iter() + .rev() + .find(|s| { + s.captured_at_ms <= now_ms + && now_ms - s.captured_at_ms <= state.policy.max_query_staleness_ms + && required.iter().all(|o| s.outputs.contains_key(&o.0)) + && eligible(&s.outputs) + }) + .ok_or(SnapshotUnavailable)?; + Ok(PinnedRevision { + revision: snapshot.revision, + captured_at_ms: snapshot.captured_at_ms, + inputs: state + .inputs + .iter() + .filter(|s| s.first_revision <= snapshot.revision) + .cloned() + .collect(), + records: required + .iter() + .flat_map(|o| snapshot.outputs[&o.0].clone()) + .collect(), + }) + } +} + +fn validate_records(output: u64, records: &[RevisionRecord]) -> Result<(), RevisionError> { + let mut keys = BTreeSet::new(); + for record in records { + record.reference.validate()?; + if record.reference.stored_output_id.0 != output + || record.start_ms >= record.end_ms + || record.payload.is_empty() + || !keys.insert((&record.group, record.start_ms, record.end_ms)) + { + return Err("invalid or duplicate revision output record".into()); + } + } + Ok(()) +} + +use crate::storage_engines::types::{ + AggregateCore, InstalledPrecomputePlanHandle, RuntimePhysicalPlan, +}; +use asap_physical_operators::{ + stored_state::native, + values::{Batch, Schema, Value}, +}; +use planner_types::post_asap::{SummaryFamilyType, SummaryField, SummarySchema}; + +fn state_schema(family: SummaryFamilyType) -> Schema { + Arc::new(SummarySchema { + fields: vec![SummaryField { + name: "state".into(), + dtype: family, + nullable: false, + }], + time_index: None, + }) +} + +pub(crate) fn encode_state( + state: Arc, + family: SummaryFamilyType, +) -> Result, RevisionError> { + Ok(native::encode_batch(&Batch::try_new( + state_schema(family.clone()), + vec![vec![Value::Summary { family, state }]], + )?)?) +} + +pub(crate) fn decode_state( + record: &RevisionRecord, + config: &asap_types::PrecomputeMaterialization, +) -> Result, RevisionError> { + let family = config.accumulator_spec()?.family; + let batch = native::decode_batch(&record.payload, state_schema(family), record.payload.len())?; + match batch.rows() { + [row] => match row.as_slice() { + [Value::Summary { state, .. }] => Ok(Arc::clone(state)), + _ => Err("revision record must contain exactly one typed summary".into()), + }, + _ => Err("revision record must contain exactly one row".into()), + } +} + +/// A single local owner serializes input revisions for the installed plan. +/// Opening is lazy so activation/recovery does not depend on receiving new data. +pub struct RevisionRuntime { + directory: PathBuf, + pub policy: RevisionPolicy, + plans: InstalledPrecomputePlanHandle, + current: Mutex)>>, +} + +impl RevisionRuntime { + pub fn new( + directory: PathBuf, + policy: RevisionPolicy, + plans: InstalledPrecomputePlanHandle, + ) -> Self { + Self { + directory, + policy, + plans, + current: Mutex::new(None), + } + } + + pub(crate) fn installed( + &self, + ) -> Result<(Arc, Arc), RevisionError> { + let plan = self + .plans + .active_physical_plan_snapshot() + .ok_or("revision execution requires an active physical plan")?; + if plan.precompute_plan.ingest.protocol + != asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 + { + return Err("continuous revisions require the local Remote Write input binding".into()); + } + if plan + .query_plan + .entries + .values() + .any(|entry| entry.instant.full_history && !entry.materialization_bindings().is_empty()) + { + return Err("continuous revisions require a bounded query history".into()); + } + let generation = plan + .precompute_plan + .summary_catalog + .as_ref() + .ok_or("revision execution requires a catalog generation")?; + let mut current = self + .current + .lock() + .map_err(|_| "revision installation poisoned")?; + if let Some((previous, store)) = current + .as_ref() + .filter(|(previous, _)| previous == generation) + { + let _ = previous; + return Ok((Arc::clone(&plan), Arc::clone(store))); + } + // A selected producer must have a typed recovery codec before any raw + // input is durably admitted. Codec failure is a deployment-format error, + // never a reason to leave an accepted input revision unrecoverable. + for config in plan + .precompute_plan + .materializations + .iter() + .filter(|c| c.derived_input.is_none()) + { + let program = super::raw_dag::RawDagProgram::from_plan(&plan.precompute_plan, config)?; + let bytes = encode_state( + Arc::from(program.updater()?.into_accumulator()), + program.family, + )?; + if bytes.len() > self.policy.max_checkpoint_bytes { + return Err(Box::new(asap_physical_operators::Error::MemoryLimit)); + } + } + let path = self.directory.join(format!( + "{}-{}.json", + generation.plan_id, generation.plan_version + )); + let store = Arc::new(RevisionStore::open( + path, + generation.clone(), + self.policy.clone(), + )?); + // Validate recovered bindings and payloads before making any result readable. + for snapshot in &store.current()?.snapshots { + for (output, records) in &snapshot.outputs { + let config = plan + .precompute_plan + .materializations + .iter() + .find(|c| c.policy_fp_u64() == *output) + .ok_or("recovered revision has an uninstalled output")?; + let reference = plan + .installed_precompute_plan + .stored_output_reference(StoredOutputId(*output)) + .ok_or("recovered revision has no installed binding")?; + for record in records { + if record.reference != reference { + return Err( + "recovered revision semantic binding differs from installed plan" + .into(), + ); + } + decode_state(record, config)?; + } + } + } + *current = Some((generation.clone(), Arc::clone(&store))); + Ok((plan, store)) + } + + pub(crate) fn query_view( + &self, + required: &BTreeSet, + now_ms: u64, + ranges: &[(StoredOutputId, u64, u64)], + expected: &CatalogGeneration, + ) -> Result { + let (plan, store) = self.installed()?; + if plan.precompute_plan.summary_catalog.as_ref() != Some(expected) { + return Err("query snapshot generation differs from the selected QueryPlan".into()); + } + let mut pinned = store.pin_matching(required, now_ms, |outputs| { + ranges + .iter() + .all(|(output, start, end)| records_cover(&outputs[&output.0], *start, *end)) + })?; + pinned.records.retain(|r| { + ranges.iter().any(|(output, start, end)| { + r.reference.stored_output_id == *output && r.start_ms >= *start && r.end_ms <= *end + }) + }); + crate::storage_engines::sketch_db::index::SketchStore::from_revision(&plan, &pinned) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn generation() -> CatalogGeneration { + CatalogGeneration { + schema_version: 6, + plan_id: 1, + plan_version: 1, + snapshot_sha256: "a".repeat(64), + } + } + fn policy() -> RevisionPolicy { + RevisionPolicy { + correction_horizon_ms: 1000, + max_query_staleness_ms: 100, + max_checkpoint_bytes: 100_000, + } + } + fn sample(time: i64, value: f64) -> InputSample { + InputSample { + metric: "x".into(), + labels: BTreeMap::new(), + series: "x".into(), + timestamp_ms: time, + first_revision: 0, + value: Some(value), + } + } + fn record(output: u64, value: u8) -> RevisionRecord { + RevisionRecord { + reference: StoredOutputReference { + stored_output_id: StoredOutputId(output), + definition_id: serde_json::from_value(serde_json::json!(format!( + "sds-v1:{}", + "a".repeat(64) + ))) + .unwrap(), + }, + group: BTreeMap::new(), + start_ms: 0, + end_ms: 100, + payload: vec![value], + } + } + fn output_set() -> BTreeSet { + BTreeSet::from([StoredOutputId(1), StoredOutputId(2)]) + } + + /// A durable partial r2 cannot force a two-branch query to mix r1 and r2. + #[test] + fn partial_revision_restart_and_pinned_read_are_consistent() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("checkpoint.json"); + let store = RevisionStore::open(path.clone(), generation(), policy()).unwrap(); + let outputs = output_set(); + store + .apply( + &generation(), + vec![sample(10, 2.)], + 100, + 1000, + &outputs, + |_, _| Ok(()), + |_, _, _, _, publish| { + publish(StoredOutputId(1), vec![record(1, 2)])?; + publish(StoredOutputId(2), vec![record(2, 2)]) + }, + ) + .unwrap(); + let old = store.pin(&outputs, 100).unwrap(); + let failure = store + .apply( + &generation(), + vec![sample(20, 3.)], + 110, + 1000, + &outputs, + |_, _| Ok(()), + |_, _, _, _, publish| { + publish(StoredOutputId(1), vec![record(1, 5)])?; + Err(Box::new(asap_physical_operators::Error::Cancelled)) + }, + ) + .unwrap_err(); + assert!(matches!( + failure.downcast_ref(), + Some(asap_physical_operators::Error::Cancelled) + )); + assert_eq!(store.pin(&outputs, 110).unwrap().revision, 1); + assert_eq!( + store + .pin(&BTreeSet::from([StoredOutputId(1)]), 110) + .unwrap() + .revision, + 2 + ); + drop(store); + let store = RevisionStore::open(path.clone(), generation(), policy()).unwrap(); + assert_eq!(store.pin(&outputs, 110).unwrap().revision, 1); + store + .apply( + &generation(), + vec![], + 120, + 1000, + &outputs, + |_, _| Ok(()), + |inputs, revision, captured, _, publish| { + assert_eq!((inputs.len(), revision, captured), (2, 2, 110)); + // Already committed bytes survive retry, including randomized sketches. + publish(StoredOutputId(1), vec![record(1, 99)])?; + publish(StoredOutputId(2), vec![record(2, 5)]) + }, + ) + .unwrap(); + let new = store.pin(&outputs, 120).unwrap(); + assert_eq!(new.revision, 2); + assert_eq!(old.inputs.len(), 1); + assert_eq!(new.inputs.len(), 2); + assert_eq!( + new.records.iter().map(|r| r.payload[0]).collect::>(), + vec![5, 5] + ); + assert_eq!( + old.records.iter().map(|r| r.payload[0]).collect::>(), + vec![2, 2] + ); + assert!(store.pin(&outputs, 211).is_err()); + drop(store); + let mut next = generation(); + next.plan_version += 1; + assert!(RevisionStore::open(path, next, policy()).is_err()); + } + + /// Newer published outputs without the requested historical pane must not + /// hide an older fresh snapshot that still has complete coverage. + #[test] + fn coverage_is_part_of_common_snapshot_selection() { + let dir = tempfile::tempdir().unwrap(); + let store = RevisionStore::open(dir.path().join("checkpoint.json"), generation(), policy()) + .unwrap(); + let outputs = output_set(); + for (time, value) in [(100, 2), (110, 3)] { + store + .apply( + &generation(), + vec![sample(time as i64 - 1, value as f64)], + time, + 1000, + &outputs, + |_, _| Ok(()), + |_, _, _, _, publish| { + for output in &outputs { + let mut record = record(output.0, value); + if time == 110 { + record.start_ms = 100; + record.end_ms = 110; + } + publish(*output, vec![record])?; + } + Ok(()) + }, + ) + .unwrap(); + } + let eligible = |records: &BTreeMap>| { + outputs + .iter() + .all(|o| records_cover(&records[&o.0], 0, 100)) + }; + assert_eq!( + store + .pin_matching(&outputs, 110, eligible) + .unwrap() + .revision, + 1 + ); + assert!(store.pin_matching(&outputs, 201, eligible).is_err()); + } + + /// Local capture must evaluate newly elapsed windows without inventing a + /// later source sample or replaying an already admitted input as new data. + #[test] + fn periodic_capture_reuses_input_without_new_admission() { + let dir = tempfile::tempdir().unwrap(); + let store = RevisionStore::open(dir.path().join("checkpoint.json"), generation(), policy()) + .unwrap(); + let outputs = output_set(); + store + .apply( + &generation(), + vec![sample(5, 2.)], + 10, + 1000, + &outputs, + |_, _| Ok(()), + |_, _, _, admitted, publish| { + assert!(admitted); + for output in &outputs { + publish(*output, vec![])?; + } + Ok(()) + }, + ) + .unwrap(); + store + .apply( + &generation(), + vec![], + 100, + 1000, + &outputs, + |_, _| Ok(()), + |inputs, revision, captured, admitted, publish| { + assert_eq!((inputs.len(), revision, captured), (1, 2, 100)); + assert!(!admitted); + for output in &outputs { + publish(*output, vec![record(output.0, 2)])?; + } + Ok(()) + }, + ) + .unwrap(); + assert_eq!(store.pin(&outputs, 100).unwrap().revision, 2); + } + + /// Rejection must not persist the valid prefix, a dedup key, or a revision. + #[test] + fn rejected_whole_batch_and_budget_leave_checkpoint_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("checkpoint.json"); + let store = RevisionStore::open(path.clone(), generation(), policy()).unwrap(); + let outputs = output_set(); + store + .apply( + &generation(), + vec![sample(10, 2.)], + 100, + 1000, + &outputs, + |_, _| Ok(()), + |_, _, _, _, publish| { + for o in &outputs { + publish(*o, vec![record(o.0, 2)])?; + } + Ok(()) + }, + ) + .unwrap(); + let before = fs::read(&path).unwrap(); + assert!(store + .apply( + &generation(), + vec![sample(20, 3.), sample(10, 99.)], + 110, + 1000, + &outputs, + |_, _| Ok(()), + |_, _, _, _, _| panic!("rejected batch executed") + ) + .is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + assert!(store + .apply( + &generation(), + vec![sample(20, 3.)], + 110, + 1000, + &outputs, + |_, _| Err("outside correction horizon".into()), + |_, _, _, _, _| panic!("rejected batch executed") + ) + .is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + let mut oversized = sample(20, 3.); + oversized.series = "x".repeat(100_000); + let error = store + .apply( + &generation(), + vec![oversized], + 110, + 1000, + &outputs, + |_, _| Ok(()), + |_, _, _, _, _| panic!("over-budget batch executed"), + ) + .unwrap_err(); + assert!(matches!( + error.downcast_ref(), + Some(asap_physical_operators::Error::MemoryLimit) + )); + assert_eq!(fs::read(&path).unwrap(), before); + } +} + +pub(crate) fn pin_query( + index: &crate::storage_engines::sketch_db::index::SketchStore, + entry: &asap_types::query_plan::QueryPlanEntry, + times: &[u64], + generation: Option<&CatalogGeneration>, +) -> Result< + Option, + crate::query_engines::EngineError, +> { + use crate::query_engines::EngineError; + let required: BTreeSet<_> = entry + .materialization_bindings() + .into_iter() + .map(|b| b.stored_output_reference.stored_output_id) + .collect(); + let current_series = entry.nodes.values().any(|node| matches!(node, asap_types::query_plan::QueryPlanNode::Logical { operator: asap_types::query_plan::residual::ResidualQueryOperator::CurrentSeries { .. }, .. })); + if required.is_empty() && !current_series { + return Ok(None); + } + let runtime = index + .revisions + .read() + .map_err(|_| { + EngineError::Physical(asap_physical_operators::Error::Invalid( + "revision installation poisoned".into(), + )) + })? + .clone(); + let Some(runtime) = runtime else { + return Ok(None); + }; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| EngineError::Physical(asap_physical_operators::Error::Invalid(e.to_string())))? + .as_millis() as u64; + let ranges: Vec<_> = entry + .materialization_bindings() + .into_iter() + .flat_map(|binding| { + times.iter().map(move |time| { + ( + binding.materialization, + time.saturating_sub( + binding + .readout_lookback_ms + .unwrap_or(entry.instant.lookback_ms), + ), + *time, + ) + }) + }) + .collect(); + runtime + .query_view( + &required, + now, + &ranges, + generation.ok_or_else(|| { + EngineError::Physical(asap_physical_operators::Error::Invalid( + "query revision requires a catalog generation".into(), + )) + })?, + ) + .map(Some) + .map_err(|error| { + if error.is::() { + return EngineError::capability_miss("summary_snapshot", error.to_string()); + } + match error.downcast::() { + Ok(native) => EngineError::Physical(*native), + Err(error) => EngineError::Physical(asap_physical_operators::Error::Invalid( + error.to_string(), + )), + } + }) +} + +fn records_cover(records: &[RevisionRecord], start: u64, end: u64) -> bool { + let mut groups = BTreeMap::<&BTreeMap, Vec<(u64, u64)>>::new(); + for record in records { + let windows = groups.entry(&record.group).or_default(); + if record.start_ms >= start && record.end_ms <= end { + windows.push((record.start_ms, record.end_ms)); + } + } + !groups.is_empty() + && groups.values_mut().all(|windows| { + windows.sort_unstable(); + let mut cursor = start; + for (lo, hi) in windows { + if *hi <= cursor { + continue; + } + if *lo != cursor { + return false; + } + cursor = *hi; + } + cursor == end + }) +} diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 01af62314..6a29e4cf0 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -10,20 +10,8 @@ use tokio::sync::mpsc; use xxhash_rust::xxh64::xxh64; /// A message sent from the router to a worker. -/// -/// B7.6 (schema-retirement #5): the per-group bucket key on `GroupSamples` -/// and `AccumulatorInput` is now a single `sid` (registry-allocated by -/// `SeriesIdResolver`), not the `(agg_id, group_key)` tuple. The grouping -/// label values are already folded into the sid via the -/// `(metric, attrs_fingerprint, agg_kind)` identity contract — so one sid -/// uniquely names one bucket, with no extra discriminator needed for -/// hashing or pane lookup. `group_key` and `policy_fp` still travel -/// alongside the sid: `group_key` is consumed at emit-time to render the -/// output label vector; `policy_fp` is the handle the worker uses to fetch -/// the source `PrecomputeMaterialization` from the hot-reload snapshot (window -/// shape, late-data policy, etc.). Together they let the worker key state -/// by sid without losing the data the legacy `(agg_id, group_key)` shape -/// carried. +/// Data partition ownership is derived from the installed DAG; storage locators +/// carried by individual inputs do not determine their destination worker. pub enum WorkerMessage { /// Immutable producer generation captured before routing. The optional /// receipt proves atomic admission; absent receipts are never fabricated. @@ -96,6 +84,11 @@ pub enum WorkerMessage { Flush, /// Finite-input barrier: acknowledge only after queued input and trailing panes reach the sink. Drain(tokio::sync::oneshot::Sender>), + /// Execute downstream DAG work after all raw windows have been sealed. + CompleteDag { + plan: Arc, + reply: tokio::sync::oneshot::Sender>, + }, /// Graceful shutdown. Shutdown, } @@ -148,6 +141,7 @@ impl fmt::Debug for WorkerMessage { .finish(), Self::Flush => f.write_str("Flush"), Self::Drain(_) => f.write_str("Drain"), + Self::CompleteDag { .. } => f.write_str("CompleteDag"), Self::Shutdown => f.write_str("Shutdown"), } } @@ -158,6 +152,7 @@ pub struct SeriesRouter { erp_observer: std::sync::OnceLock>, senders: Vec>, num_workers: usize, + plan: Option, } impl SeriesRouter { @@ -167,9 +162,25 @@ impl SeriesRouter { erp_observer: std::sync::OnceLock::new(), senders, num_workers, + plan: None, } } + pub fn with_plan( + mut self, + plan: crate::storage_engines::types::InstalledPrecomputePlanHandle, + ) -> Self { + self.plan = Some(plan); + self + } + + fn partition_rule(&self) -> super::partitioning::DagPartitioning { + self.plan + .as_ref() + .map(|plan| plan.snapshot().partitioning.clone()) + .unwrap_or(super::partitioning::DagPartitioning::Population) + } + pub fn enable_erp_observation( &self, endpoint: String, @@ -187,16 +198,15 @@ impl SeriesRouter { /// Route a pre-grouped batch of group messages to workers concurrently. /// - /// Each `GroupSamples` / `AccumulatorInput` message is routed by - /// `worker_for_sid(sid)` — same `sid` always lands on the same worker, - /// so per-bucket state stays single-owner. Messages within a single - /// worker are sent sequentially to preserve ordering. + /// Every producer uses the same DAG partition rule. Messages within one + /// worker are sent sequentially to preserve admission order. pub async fn route_group_batch( &self, messages: Vec, _ingest_received_at: Instant, generation: Option>, ) -> Result<(), Box> { + let partitioning = self.partition_rule(); // Group messages by target worker index let mut per_worker: HashMap> = HashMap::new(); for msg in messages { @@ -204,8 +214,10 @@ impl SeriesRouter { WorkerMessage::BoundInput { .. } => { return Err("input must be admitted by the router".into()) } - WorkerMessage::GroupSamples { sid, .. } => self.worker_for_sid(*sid), - WorkerMessage::AccumulatorInput { sid, .. } => self.worker_for_sid(*sid), + WorkerMessage::GroupSamples { group_key, .. } + | WorkerMessage::AccumulatorInput { group_key, .. } => { + partitioning.owner(&group_key.as_population_labels(), self.num_workers)? + } WorkerMessage::RawSamples { series_key, .. } => self.worker_for(series_key), _ => 0, }; @@ -249,16 +261,22 @@ impl SeriesRouter { String, >, ) -> Result<(), TryRouteError> { + let partitioning = self.partition_rule(); let mut pending = Vec::with_capacity(messages.len()); for message in messages { let worker_idx = match &message { - WorkerMessage::GroupSamples { sid, .. } - | WorkerMessage::AccumulatorInput { sid, .. } => self.worker_for_sid(*sid), + WorkerMessage::GroupSamples { group_key, .. } + | WorkerMessage::AccumulatorInput { group_key, .. } => partitioning + .owner(&group_key.as_population_labels(), self.num_workers) + .map_err(TryRouteError::Admission)?, WorkerMessage::RawSamples { series_key, .. } => self.worker_for(series_key), WorkerMessage::BoundInput { .. } => { return Err(TryRouteError::Admission("input already admitted".into())) } - WorkerMessage::Flush | WorkerMessage::Drain(_) | WorkerMessage::Shutdown => 0, + WorkerMessage::Flush + | WorkerMessage::Drain(_) + | WorkerMessage::Shutdown + | WorkerMessage::CompleteDag { .. } => 0, }; let permit = self.senders[worker_idx] .clone() @@ -311,14 +329,40 @@ impl SeriesRouter { Ok(()) } - /// Determine which worker handles a given sid bucket. - /// - /// Hashes the sid alone — the legacy `(agg_id, group_key)` tuple folded - /// into one u64 by `SeriesIdResolver`, so a single xxh64 over the sid - /// gives the same per-bucket sharding the tuple-hash produced. - fn worker_for_sid(&self, sid: u64) -> usize { - let hash = xxh64(&sid.to_le_bytes(), 0); - (hash as usize) % self.num_workers + pub async fn shutdown(&self) -> Result<(), String> { + let mut errors = Vec::new(); + for (worker, sender) in self.senders.iter().enumerate() { + if let Err(error) = sender.send(WorkerMessage::Shutdown).await { + errors.push(format!("worker {worker}: {error}")); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } + } + + pub async fn complete_dag( + &self, + plan: Arc, + ) -> Result<(), String> { + let mut replies = Vec::with_capacity(self.senders.len()); + for sender in &self.senders { + let (reply, receiver) = tokio::sync::oneshot::channel(); + sender + .send(WorkerMessage::CompleteDag { + plan: Arc::clone(&plan), + reply, + }) + .await + .map_err(|error| error.to_string())?; + replies.push(receiver); + } + for reply in replies { + reply.await.map_err(|error| error.to_string())??; + } + Ok(()) } /// Determine which worker handles a given series key (for raw mode). @@ -340,24 +384,54 @@ pub enum TryRouteError { #[cfg(test)] mod tests { - use super::*; + // One failed worker must not prevent shutdown of the other workers. + #[tokio::test] + async fn shutdown_reaches_healthy_workers_after_an_earlier_worker_failed() { + let (failed, receiver) = tokio::sync::mpsc::channel(1); + drop(receiver); + let (healthy, mut receiver) = tokio::sync::mpsc::channel(1); + let router = SeriesRouter::new(vec![failed, healthy]); + assert!(router.shutdown().await.is_err()); + assert!(matches!(receiver.try_recv(), Ok(WorkerMessage::Shutdown))); + } - #[test] - fn test_consistent_sid_routing() { - let (senders, _receivers): (Vec<_>, Vec<_>) = - (0..4).map(|_| mpsc::channel::(10)).unzip(); + use super::*; + // Shared DAG consumers must share partition ownership regardless of locator IDs. + #[tokio::test] + async fn shared_population_is_not_split_by_storage_locator() { + let (senders, mut receivers): (Vec<_>, Vec<_>) = + (0..4).map(|_| mpsc::channel::(128)).unzip(); let router = SeriesRouter::new(senders); - - // Same sid should always go to the same worker. - let w1 = router.worker_for_sid(42); - let w2 = router.worker_for_sid(42); - assert_eq!(w1, w2); - - // All resolved buckets land within the worker count. - assert!(router.worker_for_sid(7) < 4); - assert!(router.worker_for_sid(99) < 4); - assert!(router.worker_for_sid(0) < 4); + let messages = (0..32) + .map(|sid| WorkerMessage::GroupSamples { + sid, + policy_fp: PolicyFingerprint(sid), + group_key: Arc::new(GroupKey::new([("service", "api")])), + samples: vec![("counter".into(), 1, 1.0)], + ingest_received_at: Instant::now(), + }) + .collect(); + router + .route_group_batch(messages, Instant::now(), None) + .await + .unwrap(); + let counts = receivers + .iter_mut() + .map(|rx| { + let mut count = 0; + while rx.try_recv().is_ok() { + count += 1; + } + count + }) + .collect::>(); + assert_eq!(counts.iter().sum::(), 32); + assert_eq!( + counts.iter().filter(|n| **n != 0).count(), + 1, + "one service's complete DAG must stay with one worker: {counts:?}" + ); } #[test] diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index 660ef8906..7580daf52 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -1,8 +1,11 @@ +use asap_physical_operators::dag as execution; use asap_types::executable_plan::{BackendExecutableBinding, BackendNodeBinding}; +use futures::StreamExt; use planner_types::post_asap::PostAsapNodeId; use planner_types::post_asap::{ EdgeRole, ExecutableDag, ExecutableDagNode, ExecutableOperatorPayload, ExecutionDataState, }; +use std::{cell::RefCell, rc::Rc}; use std::{ collections::{BTreeMap, BTreeSet}, sync::Arc, @@ -28,7 +31,15 @@ pub trait PrecomputeOperatorRegistry { fn materialized_input(&self, _node: &ExecutableDagNode) -> Result, Self::Error> { Ok(None) } - fn execute(&self, node: &ExecutableDagNode, inputs: &[Arc]) -> Result; + fn output_bytes(&self, _value: &V) -> usize { + std::mem::size_of::().max(1) + } + fn execute( + &self, + node: &ExecutableDagNode, + inputs: &[Arc], + context: execution::RunContext, + ) -> Result; } /// Atomic persistence boundary. Implementations must return the already @@ -49,6 +60,7 @@ pub enum ScheduleError { Invalid(String), Operator(OperatorError), Sink(SinkError), + Execution(execution::Error), } /// Execute one precompute sink and its transitive dependencies in topological @@ -62,34 +74,64 @@ pub fn execute_precompute_sink( key: MaterializationCommitKey, registry: &R, sink: &S, + context: execution::RunContext, ) -> Result, ScheduleError> where R: PrecomputeOperatorRegistry, S: IdempotentCommitSink, { - if !matches!(binding.node(sink_node), Some(BackendNodeBinding::Materialization { stored_output }) if *stored_output == key.stored_output) - { - return Err(ScheduleError::Invalid(format!( - "commit key materialization {:?} does not match sink {}", - key.stored_output, sink_node.0 - ))); + let mut outputs = + execute_precompute_sinks(dag, binding, &[(sink_node, key)], registry, sink, context)?; + Ok(outputs.remove(0)) +} + +/// Evaluate all selected stored outputs with one dependency cache. Keys must +/// describe the same input revision and window; only their output identity may +/// differ. Validation finishes before executing or committing any output. +pub fn execute_precompute_sinks( + dag: &ExecutableDag, + binding: &BackendExecutableBinding, + outputs: &[(PostAsapNodeId, MaterializationCommitKey)], + registry: &R, + sink: &S, + context: execution::RunContext, +) -> Result>, ScheduleError> +where + R: PrecomputeOperatorRegistry, + S: IdempotentCommitSink, +{ + let mut unique = BTreeSet::new(); + for (node, key) in outputs { + if !unique.insert(node.0) + || !binding.precompute_sinks.contains(node) + || !matches!(binding.node(*node), Some(BackendNodeBinding::Materialization { stored_output }) if *stored_output == key.stored_output) + { + return Err(ScheduleError::Invalid( + "commit key does not match a unique stored output binding".into(), + )); + } + let first = &outputs[0].1; + if ( + key.plan_id, + key.plan_version, + key.window_start_ms, + key.window_end_ms, + &key.input_lineage, + ) != ( + first.plan_id, + first.plan_version, + first.window_start_ms, + first.window_end_ms, + &first.input_lineage, + ) { + return Err(ScheduleError::Invalid( + "stored outputs require one evaluation window and input revision".into(), + )); + } } binding .validate_maintenance(dag) .map_err(ScheduleError::Invalid)?; - if !binding.precompute_sinks.contains(&sink_node) - || !matches!( - binding.node(sink_node), - Some(BackendNodeBinding::Materialization { .. }) - ) - { - return Err(ScheduleError::Invalid( - "node is not a precompute sink".into(), - )); - } - if let Some(committed) = sink.get(&key).map_err(ScheduleError::Sink)? { - return Ok(committed); - } let nodes = dag .nodes .iter() @@ -131,66 +173,160 @@ where }; inputs.insert(consumer, ordered); } - let mut active = BTreeSet::new(); - let mut values = BTreeMap::>::new(); - fn visit( - id: u32, - nodes: &BTreeMap, - inputs: &BTreeMap>, - active: &mut BTreeSet, - values: &mut BTreeMap>, - registry: &R, - ) -> Result<(), ScheduleError> - where - R: PrecomputeOperatorRegistry, - { - if values.contains_key(&id) { - return Ok(()); + if outputs.is_empty() { + return Ok(Vec::new()); + } + let error = Rc::new(RefCell::new(None)); + let mut sources = BTreeMap::new(); + for (node, key) in outputs { + if let Some(value) = sink.get(key).map_err(ScheduleError::Sink)? { + sources.insert(node.0, value); } - if !active.insert(id) { - return Err(ScheduleError::Invalid("precompute subDAG cycle".into())); + } + let committed = sources.keys().copied().collect::>(); + let mut graph = execution::PhysicalDag::default(); + let mut pending = outputs.iter().map(|(id, _)| id.0).collect::>(); + let mut added = BTreeSet::new(); + while let Some(id) = pending.pop() { + if !added.insert(id) { + continue; } - let node = nodes + let node = *nodes .get(&id) .ok_or_else(|| ScheduleError::Invalid(format!("missing node {id}")))?; - if node.output_state == ExecutionDataState::QUERY_ROWS { + if node.output_state.timing == planner_types::post_asap::ExecutionTiming::QueryTime { return Err(ScheduleError::Invalid(format!( "query-time node {id} in precompute dependency path" ))); } - if let Some(value) = registry - .materialized_input(node) - .map_err(ScheduleError::Operator)? - { - values.insert(id, Arc::new(value)); - active.remove(&id); - return Ok(()); - } - let child_ids = inputs.get(&id).cloned().unwrap_or_default(); - for child in &child_ids { - visit(*child, nodes, inputs, active, values, registry)?; - } - let child_values = child_ids + let source = if let Some(value) = sources.remove(&id) { + Some(value) + } else { + registry + .materialized_input(node) + .map_err(ScheduleError::Operator)? + .map(Arc::new) + }; + let children = if source.is_some() { + vec![] + } else { + inputs.get(&id).cloned().unwrap_or_default() + }; + let schemas = children .iter() - .map(|child| Arc::clone(&values[child])) - .collect::>(); - let value = registry - .execute(node, &child_values) - .map_err(ScheduleError::Operator)?; - values.insert(id, Arc::new(value)); - active.remove(&id); - Ok(()) + .map(|child| { + nodes + .get(child) + .map(|n| n.output_schema.clone()) + .ok_or_else(|| ScheduleError::Invalid(format!("missing node {child}"))) + }) + .collect::, _>>()?; + pending.extend(children.iter().copied()); + graph + .add( + u64::from(id), + children.into_iter().map(u64::from).collect(), + IngestionOperator { + node, + registry, + source, + schemas, + error: error.clone(), + }, + ) + .map_err(ScheduleError::Execution)?; + } + let roots = outputs + .iter() + .map(|(id, _)| u64::from(id.0)) + .collect::>(); + let streams = graph + .execute(&roots, context) + .map_err(ScheduleError::Execution)?; + futures::executor::block_on(futures::future::try_join_all( + streams + .into_iter() + .zip(outputs) + .map(|(mut stream, (node, key))| { + let error = &error; + let committed = &committed; + async move { + let result = stream.next().await.ok_or_else(|| { + ScheduleError::Invalid("ingestion root produced no value".into()) + })?; + let value = match result { + Ok(value) => Arc::clone(value.value()), + Err(failure) => { + return Err(match error.borrow_mut().take() { + Some(error) => ScheduleError::Operator(error), + None => ScheduleError::Execution(failure), + }) + } + }; + if committed.contains(&node.0) { + Ok(value) + } else { + sink.commit_if_absent(key.clone(), value) + .map_err(ScheduleError::Sink) + } + } + }), + )) +} + +struct IngestionOperator<'a, V, R: PrecomputeOperatorRegistry> { + node: &'a ExecutableDagNode, + registry: &'a R, + source: Option>, + schemas: Vec, + error: Rc>>, +} +impl> + execution::PhysicalOperator, planner_types::post_asap::SummarySchema> + for IngestionOperator<'_, V, R> +{ + fn name(&self) -> &str { + "InstalledIngestionOperator" + } + fn input_schemas(&self) -> Vec { + self.schemas.clone() + } + fn output_schema(&self) -> planner_types::post_asap::SummarySchema { + self.node.output_schema.clone() + } + fn output_bytes(&self, value: &Arc) -> usize { + self.registry.output_bytes(value) + } + fn start<'a>( + &'a self, + inputs: Vec>>, + context: execution::RunContext, + ) -> Result>, execution::Error> { + Ok(futures::stream::once(async move { + if let Some(source) = &self.source { + return Ok(Arc::clone(source)); + } + let inputs = + futures::future::try_join_all(inputs.into_iter().map(|mut input| async move { + input.next().await.ok_or_else(|| { + execution::Error::Operator("ingestion input produced no value".into()) + })? + })) + .await?; + let values = inputs + .iter() + .map(|value| Arc::clone(value.value())) + .collect::>(); + self.registry + .execute(self.node, &values, context) + .map(Arc::new) + .map_err(|e| { + *self.error.borrow_mut() = Some(e); + execution::Error::Operator(format!("ingestion node {} failed", self.node.id.0)) + }) + }) + .boxed_local()) } - visit( - sink_node.0, - &nodes, - &inputs, - &mut active, - &mut values, - registry, - )?; - sink.commit_if_absent(key, values.remove(&sink_node.0).unwrap()) - .map_err(ScheduleError::Sink) } #[cfg(test)] @@ -203,6 +339,78 @@ mod tests { }; use std::sync::Mutex; + fn test_context() -> execution::RunContext { + execution::RunContext::new( + execution::Scope::Ingestion { + window_start_ms: 10, + window_end_ms: 20, + revision: 1, + }, + execution::Limits::default(), + ) + .unwrap() + } + + // Caller cancellation or exhaustion must terminate without committing state. + #[test] + fn caller_control_terminates_before_publication() { + for cancelled in [true, false] { + let context = execution::RunContext::new( + execution::Scope::Ingestion { + window_start_ms: 10, + window_end_ms: 20, + revision: 1, + }, + execution::Limits { + max_bytes: 1, + ..Default::default() + }, + ) + .unwrap(); + if cancelled { + context.cancel(); + } + let dag = ExecutableDag { + nodes: (0..4).map(node).collect(), + edges: vec![edge(0, 1), edge(1, 3)], + root: PostAsapNodeId(3), + }; + let (dag, binding) = maintenance_only(dag, binding(), PostAsapNodeId(3)); + let registry = Registry::default(); + let sink = Sink::default(); + let result = execute_precompute_sink( + &dag, + &binding, + PostAsapNodeId(3), + key(3), + ®istry, + &sink, + context, + ); + fn cause(error: &execution::Error) -> &execution::Error { + match error { + execution::Error::AtNode { source, .. } => cause(source), + error => error, + } + } + let Err(ScheduleError::Execution(error)) = result else { + panic!("expected a typed execution error, got {result:?}"); + }; + assert_eq!( + cause(&error), + if cancelled { + &execution::Error::Cancelled + } else { + &execution::Error::MemoryLimit + } + ); + assert!(sink.0.lock().unwrap().is_empty()); + if cancelled { + assert!(registry.0.lock().unwrap().is_empty()); + } + } + } + fn binding() -> BackendExecutableBinding { BackendExecutableBinding { nodes: (0..4) @@ -286,6 +494,7 @@ mod tests { &self, node: &ExecutableDagNode, inputs: &[Arc], + _context: execution::RunContext, ) -> Result { *self.0.lock().unwrap().entry(node.id.0).or_default() += 1; Ok(node.id.0 + inputs.iter().map(|v| **v).sum::()) @@ -324,6 +533,36 @@ mod tests { } } + // Two stored sinks in one evaluation reuse their shared upstream work. + #[test] + fn stored_sinks_share_one_evaluation() { + let mut query = node(4); + query.output_state = ExecutionDataState::QUERY_ROWS; + let dag = ExecutableDag { + nodes: vec![node(0), node(1), node(2), node(3), query], + edges: vec![edge(0, 1), edge(1, 2), edge(1, 3)], + root: PostAsapNodeId(4), + }; + let mut bindings = binding(); + bindings.precompute_sinks = vec![PostAsapNodeId(2), PostAsapNodeId(3)]; + let (dag, bindings) = maintenance_only(dag, bindings, PostAsapNodeId(3)); + let registry = Registry::default(); + let sink = Sink::default(); + execute_precompute_sinks( + &dag, + &bindings, + &[(PostAsapNodeId(2), key(2)), (PostAsapNodeId(3), key(3))], + ®istry, + &sink, + test_context(), + ) + .unwrap(); + assert_eq!( + *registry.0.lock().unwrap(), + BTreeMap::from([(0, 1), (1, 1), (2, 1), (3, 1)]) + ); + } + #[test] fn binary_operand_roles_survive_edge_reordering_and_reject_duplicates() { use planner_types::post_asap::BinaryOperator; @@ -335,6 +574,7 @@ mod tests { &self, node: &ExecutableDagNode, inputs: &[Arc], + _context: execution::RunContext, ) -> Result { match node.id.0 { 0 => Ok(10), @@ -375,6 +615,7 @@ mod tests { key(3), &Subtract, &Sink::default(), + test_context(), ) }; assert_eq!(*execute(&dag).unwrap(), 7); @@ -406,15 +647,29 @@ mod tests { let registry = Registry::default(); let sink = Sink::default(); let (dag, binding) = maintenance_only(dag, binding(), PostAsapNodeId(3)); - let first = - execute_precompute_sink(&dag, &binding, PostAsapNodeId(3), key(3), ®istry, &sink) - .unwrap(); + let first = execute_precompute_sink( + &dag, + &binding, + PostAsapNodeId(3), + key(3), + ®istry, + &sink, + test_context(), + ) + .unwrap(); assert_eq!(*first, 6); assert_eq!(registry.0.lock().unwrap().values().sum::(), 4); - let replay = - execute_precompute_sink(&dag, &binding, PostAsapNodeId(3), key(3), ®istry, &sink) - .unwrap(); + let replay = execute_precompute_sink( + &dag, + &binding, + PostAsapNodeId(3), + key(3), + ®istry, + &sink, + test_context(), + ) + .unwrap(); assert!(Arc::ptr_eq(&first, &replay)); assert_eq!(registry.0.lock().unwrap().values().sum::(), 4); } @@ -431,13 +686,14 @@ mod tests { &self, node: &ExecutableDagNode, inputs: &[Arc], + context: execution::RunContext, ) -> Result { assert_ne!( node.id, PostAsapNodeId(0), "absorbed source subtree must not execute" ); - self.0.execute(node, inputs) + self.0.execute(node, inputs, context) } } let mut raw = node(0); @@ -456,9 +712,16 @@ mod tests { let (dag, bindings) = maintenance_only(dag, bindings, PostAsapNodeId(3)); let registry = FrontierRegistry(Registry::default()); let sink = Sink::default(); - let result = - execute_precompute_sink(&dag, &bindings, PostAsapNodeId(3), key(3), ®istry, &sink) - .unwrap(); + let result = execute_precompute_sink( + &dag, + &bindings, + PostAsapNodeId(3), + key(3), + ®istry, + &sink, + test_context(), + ) + .unwrap(); assert_eq!(*result, 25); assert_eq!( *registry.0 .0.lock().unwrap(), @@ -499,11 +762,18 @@ mod tests { precompute_sinks: vec![PostAsapNodeId(1)], }; assert!(matches!( - execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(1), ®istry, &sink), + execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(1), ®istry, &sink, test_context()), + Err(ScheduleError::Invalid(message)) if message.contains("query-owned node") + )); + let mut summary_dag = dag.clone(); + summary_dag.nodes[0].output_state.primitive = + planner_types::post_asap::DataPrimitive::SummaryState; + assert!(matches!( + execute_precompute_sink(&summary_dag, &invalid_path_binding, PostAsapNodeId(1), key(1), ®istry, &sink, test_context()), Err(ScheduleError::Invalid(message)) if message.contains("query-owned node") )); assert!(matches!( - execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(0), ®istry, &sink), + execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(0), ®istry, &sink, test_context()), Err(ScheduleError::Invalid(message)) if message.contains("does not match") )); } diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 0bb5ebaaa..d957c0bf4 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -5,7 +5,7 @@ use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ - AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, + AggregateCore, InstalledPrecomputePlanHandle, KeyByLabelValues, PrecomputedOutput, }; #[cfg(test)] use crate::tests::accumulator_fixture::create_fixture_accumulator; @@ -39,6 +39,7 @@ use tracing::{debug, debug_span, info, warn}; struct GroupState { program: Option>, series_id: u64, + stored_output_reference: Option, catalog_generation: Option>, input_revisions: BTreeMap>, config: Arc, @@ -151,6 +152,10 @@ pub struct WorkerRuntimeConfig { /// `(metric, attrs_fingerprint, agg_kind_canonical)` identity contract on /// `SeriesIdResolver`, so one sid uniquely names one bucket. pub struct Worker { + maintenance_storage: Option<( + Arc, + Arc, + )>, erp_observer: Option>, current_input_revision: Option>, current_catalog_generation: Option>, @@ -163,7 +168,7 @@ pub struct Worker { /// Hot-reload handle — workers read config directly from ArcSwap /// instead of holding a local copy. All components see the same /// config at the same time. - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, /// Allowed lateness in ms. allowed_lateness_ms: i64, /// When true, skip aggregation and pass raw samples through. @@ -189,6 +194,48 @@ pub struct Worker { } impl Worker { + pub fn set_maintenance_storage( + &mut self, + store: Arc, + resolver: Arc, + ) { + self.maintenance_storage = Some((store, resolver)); + } + + fn complete_dag( + &self, + plan: &crate::storage_engines::types::RuntimePhysicalPlan, + ) -> Result<(), String> { + if !plan + .precompute_plan + .materializations + .iter() + .any(|output| output.derived_input.is_some()) + { + return Ok(()); + } + // The installed locality proof assigns derived graphs to one owner. + // Every raw input of this graph was routed to that same worker. + if plan.installed_precompute_plan.partitioning + != super::partitioning::DagPartitioning::SingleWorker + { + return Err("derived DAG has no validated local partitioning rule".into()); + } + if self.id != 0 { + return Ok(()); + } + let (store, resolver) = self + .maintenance_storage + .as_ref() + .ok_or("worker has no maintenance storage bindings")?; + super::maintenance_runtime::execute_finite_maintenance( + store, + resolver, + &plan.precompute_plan, + ) + .map_err(|error| error.to_string()) + } + pub fn set_erp_observer( &mut self, observer: Option>, @@ -200,7 +247,7 @@ impl Worker { id: usize, receiver: mpsc::Receiver, output_sink: Arc, - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, runtime_config: WorkerRuntimeConfig, group_count: Arc, worker_watermark: Arc, @@ -215,6 +262,7 @@ impl Worker { wall_clock_max_open_grace_period_ms, } = runtime_config; Self { + maintenance_storage: None, erp_observer: None, current_input_revision: None, current_catalog_generation: None, @@ -251,6 +299,18 @@ impl Worker { let mut processing_error: Option = None; while let Some(msg) = self.receiver.recv().await { + // A failed execution cannot publish a later batch or hide failure + // behind a successful drain. The engine must be restarted explicitly. + if let Some(error) = &processing_error { + match msg { + WorkerMessage::Drain(reply) | WorkerMessage::CompleteDag { reply, .. } => { + let _ = reply.send(Err(error.clone())); + } + WorkerMessage::Shutdown => break, + _ => {} + } + continue; + } let msg = match msg { WorkerMessage::BoundInput { input, @@ -263,6 +323,7 @@ impl Worker { { processing_error = Some("input receipt differs from captured generation".into()); + self.receiver.close(); continue; } self.current_catalog_generation = Some(generation); @@ -382,12 +443,30 @@ impl Worker { if let Err(error) = &result { processing_error = Some(error.clone()); } + if processing_error.is_some() { + self.receiver.close(); + } let _ = reply.send(processing_error.clone().map_or(Ok(()), Err)); } + WorkerMessage::CompleteDag { plan, reply } => { + let result = match &processing_error { + Some(error) => Err(error.clone()), + None => self.complete_dag(&plan), + }; + if let Err(error) = &result { + processing_error = Some(error.clone()); + } + if processing_error.is_some() { + self.receiver.close(); + } + let _ = reply.send(result); + } WorkerMessage::Shutdown => { info!("Worker {} shutting down", self.id); if let Err(e) = self.flush_all() { warn!("Worker {} final flush error: {}", self.id, e); + self.receiver.close(); + break; } // Force-close any windows still open after the final flush. // The wall-clock fallback may not yet be due for a one-shot @@ -399,6 +478,11 @@ impl Worker { break; } } + if processing_error.is_some() { + // Reject new queue reservations; otherwise ingestion could + // acknowledge data that this failed worker will never execute. + self.receiver.close(); + } } info!( @@ -416,7 +500,7 @@ impl Worker { /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// - /// Reads config directly from the `StreamingConfigHandle` + /// Reads config directly from the `InstalledPrecomputePlanHandle` /// ArcSwap handle, so new policies from a config swap are visible /// immediately — no message passing, no delay. /// Returns None if `policy_fp` has no matching config (e.g. arrived @@ -437,6 +521,7 @@ impl Worker { let gs = GroupState { program, series_id: sid, + stored_output_reference: snap.stored_output_reference(policy_fp.into()), catalog_generation: self.current_catalog_generation.clone(), input_revisions: BTreeMap::new(), window_manager: WindowManager::with_layout( @@ -643,6 +728,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, updater.take_accumulator())); debug!( @@ -719,6 +805,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, accumulator)); } @@ -850,6 +937,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, incoming.clone_boxed_core())); } @@ -898,6 +986,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, accumulator)); } @@ -916,6 +1005,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, accumulator)); } @@ -1101,6 +1191,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, accumulator)); } @@ -1118,6 +1209,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, accumulator)); } @@ -1207,6 +1299,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, accumulator)); } @@ -1224,6 +1317,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference.clone(), ); emit_batch.push((output, accumulator)); } @@ -1346,10 +1440,12 @@ fn precomputed_output_for_group( input_revisions: &BTreeMap>, series_id: u64, catalog_generation: Option<&Arc>, + stored_output_reference: Option, ) -> PrecomputedOutput { let mut output = PrecomputedOutput::new(start_timestamp, end_timestamp, Some(key), policy_fp) .with_population_labels(population_labels_from_group_key(group_key)); - output.series_id = Some(series_id); + output.storage_handle = Some(series_id); + output.stored_output_reference = stored_output_reference; output.catalog_generation = catalog_generation.cloned(); output.input_revision = i64::try_from(start_timestamp) .ok() @@ -1874,7 +1970,7 @@ mod tests { use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::output_sink::CapturingOutputSink; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_physical_operators::summary_kernels::datasketches_kll::DatasketchesKLLAccumulator; use asap_physical_operators::summary_kernels::keyed_sum_count::KeyedSumCountAccumulator; use asap_physical_operators::summary_kernels::sum::SumAccumulator; @@ -1981,16 +2077,16 @@ mod tests { ) } - /// Build a fresh `StreamingConfigHandle` from a map of agg_id + /// Build a fresh `InstalledPrecomputePlanHandle` from a map of agg_id /// → PrecomputeMaterialization. Worker::new takes this handle instead of /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( configs: HashMap, - ) -> crate::storage_engines::types::StreamingConfigHandle { - crate::storage_engines::types::StreamingConfigHandle::new( - crate::storage_engines::types::StreamingConfig::new(configs), + ) -> crate::storage_engines::types::InstalledPrecomputePlanHandle { + crate::storage_engines::types::InstalledPrecomputePlanHandle::new( + crate::storage_engines::types::InstalledPrecomputePlan::new(configs), ) } @@ -2758,14 +2854,16 @@ mod tests { } // ----------------------------------------------------------------------- - // Test: worker from streaming_config YAML + // Test: worker from installed_precompute_plan YAML // ----------------------------------------------------------------------- #[test] fn test_worker_rejects_flat_streaming_config_yaml() { let data = serde_yaml::from_str("aggregations: [{aggregationType: Sum, metric: m}]").unwrap(); - assert!(StreamingConfig::from_yaml_data(&data).is_err()); + assert!( + serde_yaml::from_value::(data).is_err() + ); } #[test] @@ -2828,6 +2926,7 @@ mod tests { &BTreeMap::new(), 1, None, + None, ); assert_eq!( output.population_labels, @@ -4085,15 +4184,16 @@ mod tests { task.await.unwrap(); } - // A sink failure remains visible on repeated barriers after panes were consumed. + // A failed publication must close admission and never publish later input. #[tokio::test] - async fn finite_input_drain_does_not_hide_sink_failure_on_retry() { - struct FailedSink; + async fn failed_worker_rejects_admission_and_never_retries_publication() { + struct FailedSink(AtomicUsize); impl OutputSink for FailedSink { fn emit_batch( &self, _: Vec<(PrecomputedOutput, Box)>, ) -> Result<(), Box> { + self.0.fetch_add(1, Ordering::SeqCst); Err("deliberate sink failure".into()) } } @@ -4113,7 +4213,8 @@ mod tests { 0, LateDataPolicy::Drop, ); - worker.output_sink = Arc::new(FailedSink); + let failed = Arc::new(FailedSink(AtomicUsize::new(0))); + worker.output_sink = failed.clone(); let (tx, rx) = tokio::sync::mpsc::channel(8); worker.receiver = rx; let task = tokio::spawn(worker.run()); @@ -4126,17 +4227,85 @@ mod tests { }) .await .unwrap(); - for _ in 0..2 { - let (ack, completed) = tokio::sync::oneshot::channel(); - tx.send(WorkerMessage::Drain(ack)).await.unwrap(); - assert!(completed - .await - .unwrap() - .unwrap_err() - .contains("deliberate sink failure")); + let (ack, completed) = tokio::sync::oneshot::channel(); + tx.send(WorkerMessage::Drain(ack)).await.unwrap(); + assert!(completed + .await + .unwrap() + .unwrap_err() + .contains("deliberate sink failure")); + assert!( + tx.send(WorkerMessage::GroupSamples { + sid: 1, + policy_fp: PolicyFingerprint(1), + group_key: test_group_key(""), + samples: group_samples("cpu", vec![(21000, 4.0)]), + ingest_received_at: std::time::Instant::now(), + }) + .await + .is_err(), + "failed workers must reject new admission" + ); + task.await.unwrap(); + assert_eq!( + failed.0.load(Ordering::SeqCst), + 1, + "a failed worker must not process later input or publish during shutdown" + ); + } + + // A failed final flush must not be followed by publishing newer open panes. + #[tokio::test] + async fn shutdown_stops_after_its_first_publication_failure() { + struct FailedSink(AtomicUsize); + impl OutputSink for FailedSink { + fn emit_batch( + &self, + _: Vec<(PrecomputedOutput, Box)>, + ) -> Result<(), Box> { + self.0.fetch_add(1, Ordering::SeqCst); + Err(Box::new(asap_physical_operators::dag::Error::MemoryLimit)) + } } + let config = make_agg_config( + 1, + "cpu", + AggregationType::SingleSubpopulation, + "Sum", + 10, + 0, + vec![], + ); + let mut worker = make_worker( + HashMap::from([(1, config)]), + Arc::new(CapturingOutputSink::new()), + false, + 0, + LateDataPolicy::Drop, + ); + let clock = Arc::new(AtomicI64::new(0)); + let read_clock = clock.clone(); + worker.set_now_ms_fn(Box::new(move || read_clock.load(Ordering::SeqCst))); + worker.wall_clock_idle_grace_period_ms = 1; + for (sid, now) in [(1, 0), (2, 10000)] { + clock.store(now, Ordering::SeqCst); + worker + .process_group_samples( + sid, + PolicyFingerprint(1), + &test_group_key(""), + group_samples("cpu", vec![(1000, 2.0)]), + ) + .unwrap(); + } + clock.store(15000, Ordering::SeqCst); + let sink = Arc::new(FailedSink(AtomicUsize::new(0))); + worker.output_sink = sink.clone(); + let (tx, rx) = tokio::sync::mpsc::channel(1); + worker.receiver = rx; tx.send(WorkerMessage::Shutdown).await.unwrap(); - task.await.unwrap(); + worker.run().await; + assert_eq!(sink.0.load(Ordering::SeqCst), 1); } #[test] @@ -4280,7 +4449,7 @@ mod tests { mod dag_execution_tests { use super::*; use crate::precompute_engine::output_sink::CapturingOutputSink; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_physical_operators::summary_kernels::exact::ExactAccumulator; use asap_types::query_plan::ExactReadout; @@ -4298,6 +4467,110 @@ mod dag_execution_tests { .unwrap() } + // Real router/engine queues preserve every partition's result as concurrency changes. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn installed_dag_results_match_with_one_and_four_workers() { + use crate::precompute_engine::{config::PrecomputeEngineConfig, engine::PrecomputeEngine}; + async fn execute(query: &str, workers: usize) -> BTreeMap<(Vec, u64, u64), Vec> { + let physical = plan(query); + assert_eq!( + physical.precompute_plan.materializations.len(), + 1, + "{query}" + ); + let config = physical.precompute_plan.materializations[0].clone(); + let runtime = + InstalledPrecomputePlan::from_precompute_plan(physical.precompute_plan).unwrap(); + let rule = runtime.partitioning.clone(); + let sink = Arc::new(CapturingOutputSink::new()); + let engine = PrecomputeEngine::new( + PrecomputeEngineConfig { + num_workers: workers, + allowed_lateness_ms: 10_000, + wall_clock_idle_grace_period_ms: 0, + wall_clock_max_open_grace_period_ms: 0, + ..Default::default() + }, + InstalledPrecomputePlanHandle::new(runtime), + sink.clone(), + Arc::new(crate::drivers::ingest::series_resolver::SeriesIdResolver::new()), + Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + ); + let ingest = engine.ingest_state(); + let diagnostics = engine.diagnostics(); + let running = tokio::spawn(engine.run()); + let mut messages = Vec::new(); + for population in 0..64 { + let label = format!("service-{population}"); + messages.push(WorkerMessage::GroupSamples { + sid: population + 1, + policy_fp: config.policy_fingerprint(), + group_key: Arc::new(GroupKey::new([("service", label.as_str())])), + samples: [ + (1000, 10.0), + (2000, 20.0), + (3000, 3.0), + (4000, 9.0), + (5000, 12.0), + ] + .into_iter() + .map(|(t, value)| { + ( + format!("{}{{service=\"{}\"}}", config.metric, label), + t, + value, + ) + }) + .collect(), + ingest_received_at: std::time::Instant::now(), + }); + } + ingest + .router + .route_group_batch(messages, std::time::Instant::now(), None) + .await + .unwrap(); + ingest.router.drain().await.unwrap(); + if workers > 1 && rule != super::super::partitioning::DagPartitioning::SingleWorker { + assert!( + diagnostics + .worker_group_counts + .iter() + .filter(|n| n.load(Ordering::Relaxed) > 0) + .count() + > 1, + "the test must exercise multiple workers" + ); + } + ingest.router.shutdown().await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), running) + .await + .unwrap() + .unwrap() + .unwrap(); + let mut records = BTreeMap::new(); + for (output, state) in sink.drain() { + let key = ( + output.key.unwrap().serialize_to_bytes(), + output.start_timestamp, + output.end_timestamp, + ); + assert!( + records.insert(key, state.serialize_to_bytes()).is_none(), + "duplicate output" + ); + } + assert!(!records.is_empty()); + records + } + for query in [ + "sum_over_time(asap_demo_gauge[5s])", + "rate(asap_demo_counter_total[5s])", + ] { + assert_eq!(execute(query, 1).await, execute(query, 4).await, "{query}"); + } + } + // A selected producer must govern updates, persisted family, and query readout. #[test] fn installed_dag_ingestion_persistence_and_readout() { @@ -4333,17 +4606,15 @@ mod dag_execution_tests { .expect("ASAP producer required") .clone(); let fp = config.policy_fingerprint(); - let streaming = StreamingConfig::from_precompute_plan(plan.precompute_plan).unwrap(); - let doc = serde_json::to_value(&streaming).unwrap(); - assert!(doc.get("aggregation_configs").is_none()); - let streaming: StreamingConfig = serde_json::from_value(doc).unwrap(); + let streaming = + InstalledPrecomputePlan::from_precompute_plan(plan.precompute_plan).unwrap(); let sink = Arc::new(CapturingOutputSink::new()); let (_tx, rx) = mpsc::channel(8); let mut worker = Worker::new( 0, rx, sink.clone(), - StreamingConfigHandle::new(streaming), + InstalledPrecomputePlanHandle::new(streaming), WorkerRuntimeConfig { max_buffer_per_series: 100, allowed_lateness_ms: 10_000, @@ -4415,13 +4686,15 @@ mod dag_execution_tests { // A flat config and a DAG whose producer no longer matches its binding cannot install. #[test] fn execution_requires_matching_dag_producer() { - assert!(serde_json::from_value::( - serde_json::json!({"aggregation_configs":{}}) - ) - .is_err()); + assert!( + serde_json::from_value::( + serde_json::json!({"aggregation_configs":{}}) + ) + .is_err() + ); let mut plan = plan("rate(asap_demo_counter_total[5s])").precompute_plan; plan.executable_dags.clear(); - assert!(StreamingConfig::from_precompute_plan(plan) + assert!(InstalledPrecomputePlan::from_precompute_plan(plan) .unwrap_err() .to_string() .contains("DAG producer")); @@ -4447,9 +4720,9 @@ mod dag_execution_tests { .unwrap(); installed.document.schema_version = asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; - let error = StreamingConfig::from_precompute_plan(plan) + assert!(InstalledPrecomputePlan::from_precompute_plan(plan) .unwrap_err() - .to_string(); - assert!(error.contains("update"), "{error}"); + .to_string() + .contains("update")); } } diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 1544a67b1..e50ae3f05 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -739,7 +739,7 @@ mod tests { .unwrap(); if seed { store.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "requests".into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::ExactAgg(AggregationType::Sum)), @@ -1045,8 +1045,8 @@ mod tests { cfg.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { name: "value".into(), }); - let hot = crate::storage_engines::types::StreamingConfigHandle::from_arc(Arc::new( - crate::storage_engines::types::StreamingConfig::new(HashMap::from([( + let hot = crate::storage_engines::types::InstalledPrecomputePlanHandle::from_arc(Arc::new( + crate::storage_engines::types::InstalledPrecomputePlan::new(HashMap::from([( cfg.policy_fp_u64(), cfg.clone(), )])), diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 05d1e8c26..9eb19a4e7 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -163,6 +163,7 @@ use asap_types::Statistic; use std::collections::HashMap; /// Simple query engine for processing PromQL-like queries against precomputed data +#[derive(Clone)] pub struct ASAPQueryEngine { #[allow(dead_code)] prometheus_scrape_interval: u64, @@ -184,6 +185,31 @@ pub struct ASAPQueryEngine { } impl ASAPQueryEngine { + /// Pin before external-input preparation and retain the view for the entire + /// request, including all branches and all range evaluation timestamps. + fn pinned_for( + &self, + physical: &crate::storage_engines::types::RuntimePhysicalPlan, + entry: &asap_types::query_plan::QueryPlanEntry, + times: &[u64], + ) -> Result, crate::query_engines::EngineError> { + let Some(index) = &self.summary_store else { + return Ok(None); + }; + let Some(view) = crate::precompute_engine::revisions::pin_query( + index, + entry, + times, + physical.precompute_plan.summary_catalog.as_ref(), + )? + else { + return Ok(None); + }; + let mut engine = self.clone(); + engine.summary_store = Some(std::sync::Arc::new(view)); + Ok(Some(engine)) + } + pub async fn execute_metricsql_at( &self, identity: &str, @@ -202,11 +228,13 @@ impl ASAPQueryEngine { .map_err(|error| { crate::query_engines::EngineError::capability_miss("query_plan", error.to_string()) })?; - let leaves = self + let pinned = self.pinned_for(&physical, planned, &[now_ms])?; + let engine = pinned.as_ref().unwrap_or(self); + let leaves = engine .prepare_query_inputs(&physical, planned, &[now_ms]) .await?; let (mut result, mut stats) = - self.execute_logical_entry(&physical, planned, &leaves, now_ms)?; + engine.execute_logical_entry(&physical, planned, &leaves, now_ms)?; stats.remote_evaluations = leaves.values().map(|leaf| leaf.remote_evaluations).sum(); stats.remote_rpcs = leaves.values().map(|leaf| leaf.remote_rpcs).sum(); annotate_logical_execution(&mut result, &stats); @@ -379,10 +407,21 @@ impl ASAPQueryEngine { crate::query_engines::EngineError, > { use crate::query_engines::EngineError; - let revision = self + let pinned = self .summary_store .as_ref() - .map(|index| index.summary_update_revision()); + .map(|index| { + crate::precompute_engine::revisions::pin_query( + index, + entry, + &[at], + physical.precompute_plan.summary_catalog.as_ref(), + ) + }) + .transpose()? + .flatten(); + let index = pinned.as_ref().or(self.summary_store.as_deref()); + let revision = index.map(|index| index.summary_update_revision()); let result = super::logical_dag::execute_installed( entry, leaves, @@ -397,7 +436,7 @@ impl ASAPQueryEngine { .. }) = entry.nodes.get(&root) { - let index = self.summary_store.as_ref().ok_or_else(|| { + let index = index.ok_or_else(|| { EngineError::capability_miss("current_series", "summary store unavailable") })?; let values = index @@ -447,7 +486,7 @@ impl ASAPQueryEngine { subtree.instant.full_history = false; subtree.instant.cumulative_readout = true; let requirement = readiness_requirement(&subtree); - let index = self.summary_store.as_ref().ok_or_else(|| { + let index = index.ok_or_else(|| { EngineError::capability_miss( "installed_logical_dag", "summary store unavailable", @@ -528,10 +567,7 @@ impl ASAPQueryEngine { )) }, ); - let current = self - .summary_store - .as_ref() - .map(|index| index.summary_update_revision()); + let current = index.map(|index| index.summary_update_revision()); if match (revision, current) { (Some(before), Some(after)) => !before.matches(after), (None, None) => false, @@ -567,13 +603,15 @@ impl ASAPQueryEngine { let times: Vec = (0..=(end - start) / step) .map(|n| start + n * step) .collect(); - let leaves = self.prepare_query_inputs(physical, entry, ×).await?; + let pinned = self.pinned_for(physical, entry, ×)?; + let engine = pinned.as_ref().unwrap_or(self); + let leaves = engine.prepare_query_inputs(physical, entry, ×).await?; let mut series = std::collections::BTreeMap::, RangeVectorElement>::new(); let mut total = super::logical_dag::ExecutionStats::default(); let mut at = start; loop { - let (result, stats) = self.execute_logical_entry(physical, entry, &leaves, at)?; + let (result, stats) = engine.execute_logical_entry(physical, entry, &leaves, at)?; total.raw_scan_evaluations += stats.raw_scan_evaluations; total.summary_readout_evaluations += stats.summary_readout_evaluations; total.memo_hits += stats.memo_hits; @@ -1005,14 +1043,16 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu { if let Some(physical) = self.active_physical_plan_snapshot() { if let Ok(entry) = physical.query_plan.lookup(query) { - let leaves = self + let pinned = self.pinned_for(&physical, entry, &[now_ms])?; + let engine = pinned.as_ref().unwrap_or(self); + let leaves = engine .prepare_query_inputs(&physical, entry, &[now_ms]) .await .map_err(|error| { tracing::warn!(query, error = %error, "installed query DAG preparation failed"); error })?; - let (mut result, mut stats) = self + let (mut result, mut stats) = engine .execute_logical_entry(&physical, entry, &leaves, now_ms) .map_err(|error| { tracing::warn!(query, error = %error, "installed query DAG execution failed"); @@ -1154,7 +1194,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu #[cfg(test)] mod sketch_query_tests { - // use crate::storage_engines::types::{CleanupPolicy, StreamingConfig}; + // use crate::storage_engines::types::{CleanupPolicy, InstalledPrecomputePlan}; // use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; // use crate::storage_engines::promsketch_store::PromSketchStore; // use crate::storage_engines::TimestampedBucketsMap; @@ -1220,13 +1260,13 @@ mod sketch_query_tests { // let inference_config = // InferenceConfig::new(::promql, CleanupPolicy::NoCleanup); - // let streaming_config = Arc::new(StreamingConfig::default()); + // let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); // ASAPQueryEngine::new( // Arc::new(NoOpStore), // Some(ps), // inference_config, - // streaming_config, + // installed_precompute_plan, // 15, // ::promql, // ) @@ -1296,11 +1336,11 @@ mod sketch_query_tests { // // Engine with promsketch_store = None // let inference_config = // InferenceConfig::new(::promql, CleanupPolicy::NoCleanup); - // let streaming_config = Arc::new(StreamingConfig::default()); + // let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); // let engine = ASAPQueryEngine::new( // Arc::new(NoOpStore), // inference_config, - // streaming_config, + // installed_precompute_plan, // 15, // ::promql, // ); @@ -1367,11 +1407,11 @@ mod sketch_query_tests { // fn test_sketch_range_returns_none_without_store() { // let inference_config = // InferenceConfig::new(::promql, CleanupPolicy::NoCleanup); - // let streaming_config = Arc::new(StreamingConfig::default()); + // let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); // let engine = ASAPQueryEngine::new( // Arc::new(NoOpStore), // inference_config, - // streaming_config, + // installed_precompute_plan, // 15, // ::promql, // ); @@ -1470,11 +1510,11 @@ mod aux_pushdown_tests { fn make_engine() -> ASAPQueryEngine { use crate::storage_engines::types::{ - CleanupPolicy, StreamingConfig, StreamingConfigHandle, + CleanupPolicy, InstalledPrecomputePlan, InstalledPrecomputePlanHandle, }; - let sc = Arc::new(StreamingConfig::new(HashMap::new())); - let hr = StreamingConfigHandle::from_arc(sc.clone()); + let sc = Arc::new(InstalledPrecomputePlan::new(HashMap::new())); + let hr = InstalledPrecomputePlanHandle::from_arc(sc.clone()); let _ = sc; ASAPQueryEngine::new(60) } @@ -1602,7 +1642,7 @@ mod asap_tier_classify_tests { AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::{CleanupPolicy, StreamingConfigHandle}; + use crate::storage_engines::types::{CleanupPolicy, InstalledPrecomputePlanHandle}; use std::collections::{BTreeMap, BTreeSet}; /// `sum by (zone) (http_requests_total)` end-to-end via the @@ -1634,7 +1674,7 @@ mod asap_tier_classify_tests { for (i, zone) in zones.iter().enumerate() { let sid = 9000 + i as u64; idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), capability: Some(Capability::ExactAgg(AggregationType::Sum)), @@ -1725,7 +1765,7 @@ mod asap_tier_classify_tests { // Latest ASAPPlanner sizes an epsilon=0.01 KLL at k=269. let cfg = SketchConfig::Kll { k: 269 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), @@ -1763,7 +1803,7 @@ mod asap_tier_classify_tests { // Latest ASAPPlanner requires p=14 for a 1% HLL error target. let cfg = SketchConfig::Hll { precision: 14 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::CardinalityApprox), @@ -2150,7 +2190,11 @@ mod asap_tier_classify_tests { query: QueryReadout::Quantile { q: 0.99 }, }, ); - let sids = idx.snapshot_instances().iter().map(|m| m.sid).collect(); + let sids = idx + .snapshot_instances() + .iter() + .map(|m| m.storage_handle) + .collect(); let engine = test_plan::engine(idx, config, sids, entry); engine.execute_at(query, now_ms).await.expect( "quantile_over_time over a Hit KLL sid must NOT capability-miss \ @@ -2280,7 +2324,7 @@ mod asap_tier_classify_tests { for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 14_000 + i as u64; idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), capability: Some(Capability::ExactAgg(AggregationType::Sum)), @@ -2376,7 +2420,7 @@ mod asap_tier_classify_tests { // Matches ControlPlaneCostModel's epsilon=0.01 CMS sizing. let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() @@ -2491,7 +2535,7 @@ mod outer_agg_integration_tests { AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::StreamingConfigHandle; + use crate::storage_engines::types::InstalledPrecomputePlanHandle; use asap_sketchlib::DdSketch; use asap_sketchlib::MessagePackCodec; use std::collections::{BTreeMap, BTreeSet}; @@ -2512,7 +2556,7 @@ mod outer_agg_integration_tests { relative_accuracy: 0.01, }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() @@ -2619,7 +2663,7 @@ mod range_stitch_tests { AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::{KeyByLabelValues, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlanHandle, KeyByLabelValues}; use async_trait::async_trait; use std::collections::{BTreeMap, BTreeSet}; @@ -2653,7 +2697,7 @@ mod range_stitch_tests { fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), @@ -2778,7 +2822,7 @@ mod range_stitch_tests { .unwrap(); active.envelope.expiry_unix_ms = None; let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new(active); - let hot = StreamingConfigHandle::from_active_physical_plan(active.clone()); + let hot = InstalledPrecomputePlanHandle::from_active_physical_plan(active.clone()); let engine = ASAPQueryEngine::new(15).with_active_physical_plan(active); let error = engine .execute_metricsql_at(&identity, 1_000) diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index aed8dd8b9..8e9fe425b 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -932,7 +932,7 @@ mod tests { const MATERIALIZATION: asap_types::PolicyFingerprint = asap_types::PolicyFingerprint(9001); let store = crate::storage_engines::sketch_db::index::SketchStore::new(); store.register(SummarySeriesMetadata { - sid: 41, + storage_handle: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 42f0974f8..8c9e26466 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -150,7 +150,7 @@ mod tests { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(901); idx.register(SummarySeriesMetadata { - sid: 9, + storage_handle: 9, metric_name: "bytes".into(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 35f6e22c6..cb6c25023 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -541,6 +541,48 @@ fn execute_physical_query_payload( t1_ms: u64, is_cumulative: bool, ) -> Result { + // Pin once for every branch; publication cannot switch a later read to r2. + let revisions = index + .revisions + .read() + .map_err(|_| LoweringSkip::ExecuteFailed("revision installation poisoned".into()))? + .clone(); + let view = if let Some(revisions) = revisions { + let required = entry + .materialization_bindings() + .into_iter() + .map(|b| b.stored_output_reference.stored_output_id) + .collect(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| LoweringSkip::ExecuteFailed(e.to_string()))? + .as_millis() as u64; + let ranges: Vec<_> = entry + .materialization_bindings() + .into_iter() + .map(|b| (b.materialization, t0_ms, t1_ms)) + .collect(); + Some( + revisions + .query_view( + &required, + now, + &ranges, + index + .active_catalog_generation() + .as_deref() + .ok_or_else(|| { + LoweringSkip::ExecuteFailed( + "query revision requires a catalog generation".into(), + ) + })?, + ) + .map_err(|e| LoweringSkip::ExecuteFailed(e.to_string()))?, + ) + } else { + None + }; + let index = view.as_ref().unwrap_or(index); let revision = index.summary_update_revision(); let result = (|| { let runtime = PhysicalQueryRuntime { @@ -758,7 +800,7 @@ mod tests { let mut group_by_keys = std::collections::BTreeSet::new(); group_by_keys.insert("service".to_string()); idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "unique_users".to_string(), group_by_keys, capability: Some(Capability::CardinalityApprox), @@ -800,7 +842,7 @@ mod tests { relative_accuracy: 0.01, }; idx.register(SummarySeriesMetadata { - sid: 1, + storage_handle: 1, metric_name: "latency_ms".to_string(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), @@ -1030,7 +1072,7 @@ mod tests { let idx = SketchStore::new(); idx.register( crate::storage_engines::sketch_db::index::SummarySeriesMetadata { - sid: 1, + storage_handle: 1, metric_name: "bytes_total".to_string(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1129,7 +1171,7 @@ mod tests { idx.install_summary_catalog(std::sync::Arc::new(plan.summary_catalog.clone())) .unwrap(); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1201,7 +1243,7 @@ mod tests { idx.install_summary_catalog(std::sync::Arc::new(plan.summary_catalog.clone())) .unwrap(); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1271,7 +1313,7 @@ mod tests { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1374,7 +1416,7 @@ mod tests { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 7aa36ac0c..356a1a2f3 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -520,7 +520,7 @@ impl QueryExecutionContext<'_> { }; let mut sids = self .index - .series_ids_for_policy(binding.materialization.fingerprint()); + .storage_handles_for_output(&binding.stored_output_reference); sids.sort_unstable(); sids.dedup(); let mut matched_metadata = 0usize; @@ -1652,7 +1652,7 @@ mod tests { fn kll_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() @@ -1675,7 +1675,7 @@ mod tests { fn hll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::Hll { precision: 10 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::CardinalityApprox), @@ -1721,7 +1721,7 @@ mod tests { fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), @@ -1794,7 +1794,7 @@ mod tests { fn cms_with_heap_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::FrequencyTopk(Some( @@ -1864,7 +1864,7 @@ mod tests { fn sum_exact_agg_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index 563d85a1b..ef0a54c52 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -178,7 +178,7 @@ pub(super) fn bound_reference( .iter() .filter(|m| !m.policy_fp.is_unset()) .map(|m| { - let (summary, data) = index.descriptors_for_series_id(m.sid).unwrap(); + let (summary, data) = index.descriptors_for_series_id(m.storage_handle).unwrap(); (m.policy_fp, (*summary).clone(), (*data).clone()) }) .collect::>(); diff --git a/data_plane/src/query_engines/routing/backend_storage_routing.rs b/data_plane/src/query_engines/routing/backend_storage_routing.rs index bbae8d30f..e3aa2b397 100644 --- a/data_plane/src/query_engines/routing/backend_storage_routing.rs +++ b/data_plane/src/query_engines/routing/backend_storage_routing.rs @@ -804,7 +804,7 @@ pub fn routing_table_hash(table: &BackendStorageRouting) -> String { // --------------------------------------------------------------------------- /// Per-tenant atomic-swap wrapper around `BackendStorageRouting`, -/// mirroring [`crate::storage_engines::types::StreamingConfigHandle`]. Lets the +/// mirroring [`crate::storage_engines::types::InstalledPrecomputePlanHandle`]. Lets the /// `POST /api/v1/storage_routing` HTTP handler swap one tenant's table /// at runtime without restarting the backend or touching any other /// tenant's table. Cloneable; clones share the underlying `ArcSwap` so diff --git a/data_plane/src/query_engines/routing/capability_matching.rs b/data_plane/src/query_engines/routing/capability_matching.rs index 379f3c3f5..ca588a650 100644 --- a/data_plane/src/query_engines/routing/capability_matching.rs +++ b/data_plane/src/query_engines/routing/capability_matching.rs @@ -4,7 +4,7 @@ //! //! Split out of `asap_types`'s former `capability_matching` module (see //! `scratchpad/artifacts/enum-unification-plan.md`). `StorageBackend` and -//! the `StreamingConfig` wire format it's a field of both turned out to +//! the `InstalledPrecomputePlan` wire format it's a field of both turned out to //! have zero real `control_plane` dependency either — see //! [`crate::storage_engines::types::storage_backend`]'s module doc — so //! both moved into this crate; only the routing *decision* below was ever diff --git a/data_plane/src/query_engines/routing/mod.rs b/data_plane/src/query_engines/routing/mod.rs index 3bf7ad70b..caa5f5548 100644 --- a/data_plane/src/query_engines/routing/mod.rs +++ b/data_plane/src/query_engines/routing/mod.rs @@ -18,7 +18,7 @@ //! * [`capability_matching`] — the storage-backend routing policy itself //! (`AccuracyTarget`, `compatible_storage_backends`). Split out of //! `asap_types`'s former `capability_matching` module. `StorageBackend` -//! itself later moved into this crate too, alongside `StreamingConfig` +//! itself later moved into this crate too, alongside `InstalledPrecomputePlan` //! (see `crate::storage_engines::types::storage_backend`'s module doc) — //! `control_plane` turned out to have zero real dependency on either. //! diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index ce8e476d2..9dacddcfb 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -491,7 +491,7 @@ impl BackfillRegistry { /// world there is no `SchemaRegistry::get(agg_id)` to look /// it up from, and the caller (typically the HTTP handler /// or control plane) already has the wall-clock snapshot in - /// scope from its `StreamingConfig` reconcile event. + /// scope from its `InstalledPrecomputePlan` reconcile event. /// * **Within data retention** (if `data_retention_ms` is /// provided): `time_range.0 >= now - data_retention_ms`. /// Method B from the design discussion — fail fast instead diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index b38df3268..ceb6e9844 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -21,7 +21,9 @@ use tracing::debug; use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; +use crate::storage_engines::types::{ + AggregateCore, InstalledPrecomputePlanHandle, KeyByLabelValues, +}; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; @@ -87,22 +89,26 @@ fn resolve_backfill_bucket_sid( .map(|name| (name.as_str(), *labels.get(name.as_str()).unwrap_or(&""))) .collect(); let attrs_fp = population_attrs_fingerprint(config.population_key_encoding, &grouping_pairs)?; - let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); - resolver.resolve_with_reactivation(&config.metric, &attrs_fp, &agg_kind_canonical, |sid| { - store.map_or(Ok(None), |store| { - store.validate_routed_catalog_generation(captured_generation)?; - let activation = - store.authorize_series_reactivation(sid, config.policy_fingerprint().into())?; - if activation - .as_deref() - .is_some_and(|generation| Some(generation) != captured_generation) - { - return Err("stale backfill job cannot reactivate series".into()); - } - Ok(activation) - }) - }) + if let Some(store) = store { + return store.resolve_output_storage_handle( + resolver, + config.policy_fingerprint().into(), + &attrs_fp, + captured_generation, + ); + } + #[cfg(test)] + { + let store = crate::storage_engines::sketch_db::index::SketchStore::new(); + store.resolve_output_storage_handle( + resolver, + config.policy_fingerprint().into(), + &attrs_fp, + captured_generation, + ) + } + #[cfg(not(test))] + Err("backfill output requires an installed SummaryStore".into()) } /// Fallback bucket id for the resolver-less code path (registry-only @@ -124,10 +130,10 @@ fn fallback_bucket_id(group_key: &str) -> u64 { /// without round-tripping through the worker. pub struct BackfillWindowProcessor { /// Live config source. The processor snapshots the latest - /// `StreamingConfig` at each window to find the + /// `InstalledPrecomputePlan` at each window to find the /// `PrecomputeMaterialization` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. - config: StreamingConfigHandle, + config: InstalledPrecomputePlanHandle, /// Destination for rebuilt windows. Tests may omit it to record registry /// provenance without storing payloads. summary_store: Option>, @@ -149,7 +155,7 @@ pub struct BackfillWindowProcessor { impl BackfillWindowProcessor { pub fn new( - config: StreamingConfigHandle, + config: InstalledPrecomputePlanHandle, registry: Arc, job_id: u64, ) -> Self { @@ -208,7 +214,7 @@ impl WindowProcessor for BackfillWindowProcessor { let config = snapshot .get_aggregation_config(agg_id) .cloned() - .ok_or_else(|| format!("agg_id {agg_id} not in current StreamingConfig"))?; + .ok_or_else(|| format!("agg_id {agg_id} not in current InstalledPrecomputePlan"))?; let program = snapshot.raw_programs.get(&agg_id).cloned(); #[cfg(not(test))] if program.is_none() { @@ -312,7 +318,9 @@ impl WindowProcessor for BackfillWindowProcessor { self.job_id, PolicyFingerprint::from_config(&config), ); - output.series_id = Some(sid); + output.storage_handle = Some(sid); + output.stored_output_reference = + snapshot.stored_output_reference(config.policy_fingerprint().into()); output.catalog_generation = self.catalog_generation.clone(); batch.push((sid, output, accumulator)); } @@ -329,13 +337,40 @@ impl WindowProcessor for BackfillWindowProcessor { // to the same sid (the resolver is idempotent), // but the round-trip is redundant now that we hold // the value. - for (sid, output, accumulator) in &batch { - idx.ingest_precompute_with_series_id( - *sid, - &config, - output, - accumulator.as_ref(), - ) + for (_handle, output, accumulator) in &batch { + let reference = output + .stored_output_reference + .clone() + .ok_or("backfill has no selected stored output")?; + let (plan_id, plan_version) = self + .catalog_generation + .as_deref() + .map(|generation| (generation.plan_id, generation.plan_version)) + .unwrap_or((0, 0)); + let address = asap_types::sds::StoredSummaryKey { + plan_id, + plan_version, + stored_output_id: reference.stored_output_id, + population: config + .grouping_labels + .iter() + .cloned() + .zip(output.key.clone().unwrap_or_default().labels) + .collect(), + window: asap_types::sds::HalfOpenTimeRange { + start_ms: i64::try_from(output.start_timestamp)?, + end_ms: i64::try_from(output.end_timestamp)?, + }, + }; + idx.publish_unadmitted_summary_update(|writer| { + writer.write_stored_summary( + &address, + _resolver, + &config, + output, + accumulator.as_ref(), + ) + }) .ok_or("backfill summary state publication rejected")?; } } @@ -373,7 +408,7 @@ mod tests { }; use crate::storage_engines::sketch_db::backfill::worker::BackfillWorker; use crate::storage_engines::sketch_db::backfill::BackfillSource; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -406,10 +441,10 @@ mod tests { ) } - fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { + fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { let mut map = std::collections::HashMap::new(); map.insert(config.policy_fp_u64(), config); - Arc::new(StreamingConfig::new(map)) + Arc::new(InstalledPrecomputePlan::new(map)) } #[tokio::test] @@ -417,7 +452,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -461,7 +496,7 @@ mod tests { async fn unknown_agg_id_fails_cleanly() { let cfg = sum_config(1, "m", vec![]); let streaming = streaming_config_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( 999, @@ -470,12 +505,14 @@ mod tests { 1, ); let processor = BackfillWindowProcessor::new(hot, registry.clone(), job_id); - // agg_id=999 isn't in the StreamingConfig. + // agg_id=999 isn't in the InstalledPrecomputePlan. let err = processor .process_window(999, (0, 10), vec![]) .await .expect_err("unknown agg should fail"); - assert!(err.to_string().contains("not in current StreamingConfig")); + assert!(err + .to_string() + .contains("not in current InstalledPrecomputePlan")); assert!(registry.windows_written_by(job_id).is_empty()); } @@ -484,7 +521,7 @@ mod tests { let cfg = sum_config(1, "m", vec![]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -505,7 +542,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -797,7 +834,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let summary_store = Arc::new(SketchStore::new()); let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( @@ -863,10 +900,23 @@ mod tests { // `SeriesIdResolver::lookup` would return for the same // `(metric, grouping-values, agg_kind)` tuple — i.e. live // ingest and backfill share one sid namespace. - let sid_a = - resolve_backfill_bucket_sid(&resolver, &cfg, "latency{svc=\"a\"}", None, None).unwrap(); - let sid_b = - resolve_backfill_bucket_sid(&resolver, &cfg, "latency{svc=\"b\"}", None, None).unwrap(); + let generation = summary_store.active_catalog_generation().unwrap(); + let sid_a = resolve_backfill_bucket_sid( + &resolver, + &cfg, + "latency{svc=\"a\"}", + Some(&summary_store), + Some(&generation), + ) + .unwrap(); + let sid_b = resolve_backfill_bucket_sid( + &resolver, + &cfg, + "latency{svc=\"b\"}", + Some(&summary_store), + Some(&generation), + ) + .unwrap(); assert_ne!(sid_a, sid_b, "distinct svc values mint distinct sids"); assert_eq!(summary_store.classify(sid_a), SeriesLookup::Hit); assert_eq!(summary_store.classify(sid_b), SeriesLookup::Hit); @@ -905,11 +955,10 @@ mod tests { ); let attrs = population_attrs_fingerprint(encoding, &[("svc", "a"), ("zone", "z0")]).unwrap(); - let expected_sid = resolver.resolve( - &cfg.metric, - &attrs, - &crate::storage_engines::sketch_db::data::materialization_kind_for_config(&cfg), - ); + let store = crate::storage_engines::sketch_db::index::SketchStore::new(); + let expected_sid = store + .resolve_output_storage_handle(&resolver, cfg.policy_fingerprint().into(), &attrs, None) + .unwrap(); if encoding.is_legacy() { assert_eq!(backfill_result.unwrap(), expected_sid); } else { @@ -934,7 +983,16 @@ mod tests { let acc = asap_physical_operators::summary_kernels::sum::SumAccumulator::with_sum(1.0); let live_sid = store .ingest_precompute_for_agg_config( - |metric, attrs, kind| resolver.resolve(metric, attrs, kind), + |_metric, attrs, _kind| { + store + .resolve_output_storage_handle( + &resolver, + cfg.policy_fingerprint().into(), + attrs, + None, + ) + .ok() + }, &cfg, &output, &acc, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index 86d64fac3..35c9b1353 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -58,7 +58,7 @@ use crate::storage_engines::sketch_db::backfill::worker::BackfillWorker; use crate::storage_engines::sketch_db::backfill::{ BackfillRegistry, BackfillSource, BackfillStatus, }; -use crate::storage_engines::types::StreamingConfigHandle; +use crate::storage_engines::types::InstalledPrecomputePlanHandle; /// Given a `BackfillSource`, return a reader that can read raw /// samples from it. Used by the service to pick a concrete reader @@ -103,7 +103,7 @@ pub struct BackfillService { /// Shared sid mint authority — wired alongside `sketch_index` so /// backfilled precompute sids share the namespace with live ingest. series_resolver: Option>, - config_source: StreamingConfigHandle, + config_source: InstalledPrecomputePlanHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, } @@ -111,7 +111,7 @@ pub struct BackfillService { impl BackfillService { pub fn new( registry: Arc, - config_source: StreamingConfigHandle, + config_source: InstalledPrecomputePlanHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, ) -> Self { @@ -341,7 +341,7 @@ mod tests { use crate::storage_engines::sketch_db::backfill::raw_sample_reader::{ MockRawSampleReader, RawSample, }; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; @@ -369,10 +369,10 @@ mod tests { ) } - fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { + fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { let mut m = std::collections::HashMap::new(); m.insert(cfg.policy_fp_u64(), cfg); - Arc::new(StreamingConfig::new(m)) + Arc::new(InstalledPrecomputePlan::new(m)) } async fn wait_for_status( @@ -405,7 +405,7 @@ mod tests { let mut cfg = sum_config(1, "latency"); cfg.table_name = Some("expected_table".into()); let agg_fp = cfg.policy_fp_u64(); - let hot = StreamingConfigHandle::from_arc(streaming_with(cfg)); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming_with(cfg)); let registry = Arc::new(BackfillRegistry::new()); let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let called = calls.clone(); @@ -441,7 +441,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory returns a fresh mock reader per call — seeded with a @@ -488,7 +488,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( @@ -522,7 +522,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory records the order in which it's invoked. @@ -584,7 +584,7 @@ mod tests { async fn service_shutdown_stops_the_loop() { let cfg = sum_config(1, "m"); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index 1d59bb02c..3224b7248 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -7,7 +7,8 @@ use super::*; use crate::storage_engines::types::AggregateCore; pub(crate) struct FrozenExactWindows { - pub(crate) sid: u64, + pub(crate) stored_output_reference: asap_types::sds::StoredOutputReference, + pub(crate) storage_handle: u64, pub(crate) definition: StoredOutputId, pub(crate) generation: Arc, pub(crate) group: BTreeMap, @@ -122,12 +123,15 @@ impl SketchStore { let writer = metadata .as_ref() .ok_or("immutable population proof requires durable metadata")?; + // Retained populations remain a completeness fence even when their + // generation is no longer readable. Losing their binding cannot certify + // that a newly observed subset is the complete source population. Ok(writer .load_strict() .map_err(|error| error.to_string())? .into_iter() .filter(|record| !record.removed && record.stored_output_id == Some(definition)) - .map(|record| record.sid) + .map(|record| record.storage_handle) .collect()) } @@ -183,7 +187,12 @@ impl SketchStore { population_ids.extend( instances .iter() - .filter(|(_, binding)| binding.metadata.policy_fp == definition.fingerprint()) + .filter(|(_, binding)| { + binding.metadata.policy_fp == definition.fingerprint() + && binding.stored_output_reference + == self.descriptors.stored_output_reference(definition) + && binding.catalog_generation.as_deref() == Some(generation) + }) .map(|(sid, _)| *sid), ); if !require_complete_population && population_ids.len() > 1 { @@ -216,6 +225,8 @@ impl SketchStore { let mut coordinates = BTreeMap::new(); for (sid, binding) in instances.iter() { if binding.metadata.policy_fp != definition.fingerprint() + || binding.stored_output_reference + != self.descriptors.stored_output_reference(definition) || binding.catalog_generation.as_deref() != Some(generation) || !binding.metadata.is_writable() { @@ -285,13 +296,15 @@ impl SketchStore { .admission .read() .map_err(|_| "admission registry poisoned")?; - let (keys, singleton_population_complete) = { + let (keys, singleton_population_complete, stored_output_reference) = { let bindings = self .instances .read() .map_err(|_| "instance registry poisoned")?; let binding = bindings.get(&sid).ok_or("immutable input SID is absent")?; if binding.metadata.policy_fp != definition.fingerprint() + || binding.stored_output_reference + != self.descriptors.stored_output_reference(definition) || binding.catalog_generation.as_deref() != Some(generation.as_ref()) || binding.metadata.status() == AggStatus::Expired { @@ -303,6 +316,9 @@ impl SketchStore { .iter() .filter(|(_, candidate)| { candidate.metadata.policy_fp == definition.fingerprint() + && candidate.stored_output_reference + == self.descriptors.stored_output_reference(definition) + && candidate.catalog_generation.as_deref() == Some(generation.as_ref()) }) .map(|(sid, _)| *sid), ); @@ -312,7 +328,14 @@ impl SketchStore { asap_types::sds::DataSourceIdentity::Derived { .. } ) && population_ids == BTreeSet::from([sid]); - (binding.metadata.group_by_keys.clone(), singleton) + ( + binding.metadata.group_by_keys.clone(), + singleton, + binding + .stored_output_reference + .clone() + .ok_or("immutable input has no stored-output binding")?, + ) }; drop(admission); if group.keys().cloned().collect::>() != keys { @@ -371,7 +394,8 @@ impl SketchStore { } self.validate_routed_catalog_generation(Some(generation.as_ref()))?; Ok(FrozenExactWindows { - sid, + stored_output_reference, + storage_handle: sid, definition, generation: Arc::clone(generation), group: group.clone(), @@ -406,16 +430,17 @@ impl SketchStore { let mut populations = BTreeSet::new(); for source in sources { if &source.generation != generation - || !populations.insert((source.definition, source.sid, &source.group)) + || !populations.insert((source.definition, source.storage_handle, &source.group)) { return Err( "immutable publication input cohort has mixed generations or duplicates".into(), ); } let binding = instances - .get(&source.sid) + .get(&source.storage_handle) .ok_or("immutable source was removed")?; - if binding.metadata.policy_fp != source.definition.fingerprint() + if binding.stored_output_reference != Some(source.stored_output_reference.clone()) + || binding.metadata.policy_fp != source.definition.fingerprint() || binding.catalog_generation.as_deref() != Some(generation.as_ref()) || !binding.metadata.is_writable() { @@ -440,14 +465,23 @@ impl SketchStore { supplied .entry(input.definition) .or_default() - .insert(input.sid); + .insert(input.storage_handle); } for (definition, supplied_sids) in supplied { let mut current = self.durable_maintenance_population_ids(definition)?; current.extend( instances .iter() - .filter(|(_, binding)| binding.metadata.policy_fp == definition.fingerprint()) + .filter(|(_, binding)| { + binding.metadata.policy_fp == definition.fingerprint() + && binding.stored_output_reference + == self.descriptors.stored_output_reference(definition) + && binding.catalog_generation.as_deref() + == cohort + .inputs() + .first() + .map(|input| input.generation.as_ref()) + }) .map(|(sid, _)| *sid), ); if current != supplied_sids { @@ -1042,7 +1076,7 @@ mod tests { .load_strict() .unwrap() .iter() - .any(|record| record.sid == 905)); + .any(|record| record.storage_handle == 905)); store.force_expire(901).unwrap(); assert!(store .publish_frozen_maintenance_output(903, target, &output, &sum, &cohort, [42; 32]) @@ -1061,7 +1095,7 @@ mod tests { .load_strict() .unwrap() .iter() - .any(|record| record.sid == 903)); + .any(|record| record.storage_handle == 903)); let mut next = catalog; next.plan_version += 1; store.install_summary_catalog(Arc::new(next)).unwrap(); @@ -1377,7 +1411,7 @@ mod tests { .load_strict() .unwrap() .into_iter() - .map(|record| (record.sid, serde_json::to_value(record).unwrap())) + .map(|record| (record.storage_handle, serde_json::to_value(record).unwrap())) .collect::>() }; let records_before = durable_records(); @@ -1468,7 +1502,11 @@ mod tests { .unwrap(); assert!(!store.completed_windows.read().unwrap().contains_key(&601)); let input = FrozenExactWindows { - sid: 600, + stored_output_reference: store + .descriptors + .stored_output_reference(source_id) + .unwrap(), + storage_handle: 600, definition: source_id, generation, group: BTreeMap::new(), diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 3c9dc4812..c3874750d 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -19,6 +19,7 @@ //! See design doc §4.6 ("OTLP metadata model + backend store layout") at //! `docs/design_docs/series-identity.md`. +use asap_types::sds::StoredOutputReference; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -194,14 +195,14 @@ fn build_attrs_fp_and_label_map( Ok((attrs_fp, label_values_map)) } -/// Metadata for one logical sketch instance, keyed by sid. Ingest registers it -/// on first sight; later writes append to the associated per-sid storage. +/// Physical metadata for one stored-output population. Its local handle locates +/// rows; the owning `SdsBinding` retains the selected output and plan generation. /// /// Lifecycle status is derived from retirement and expiry timestamps plus the /// wall clock. Only active instances accept writes. #[derive(Debug, Clone)] pub struct SummarySeriesMetadata { - pub sid: u64, + pub storage_handle: u64, pub metric_name: String, /// The group-by KEY set — `dp.attributes.keys()` after the agent's /// `AggregateBy` rollup folded other labels into the sketch state. @@ -233,7 +234,7 @@ pub struct SummarySeriesMetadata { /// `AggSchema::expires_at_ms`. pub expires_at_ms: Option, /// Content-addressed back-reference to the policy that minted this - /// sid. Together with [`SketchStore::policy_to_series_ids`] this gives + /// sid. Together with [`SketchStore::output_to_storage_handles`] this gives /// the query path a direct `policy_fp → [sid]` index without /// walking the metadata map. `PolicyFingerprint::UNSET` is reserved /// for the legacy registration path that doesn't carry a source @@ -608,6 +609,7 @@ impl Drop for StateMutation<'_> { #[derive(Default)] pub struct SketchStore { + pub(crate) revisions: RwLock>>, pub current_series: std::sync::Mutex, /// Held through each state append; completion takes the exclusive guard. completed_windows: RwLock>, @@ -654,32 +656,32 @@ pub struct SketchStore { /// /// ## Cross-index atomicity invariant (P2-1) /// - /// `instances`, `policy_to_series_ids`, and `metric_to_series_ids` form ONE + /// `instances`, `output_to_storage_handles`, and `metric_to_series_ids` form ONE /// logical index whose three maps must agree: every sid present in /// `instances` must also be present in `metric_to_series_ids` (keyed by /// its metric_name) and — when its `policy_fp` is non-UNSET — in - /// `policy_to_series_ids`. A concurrent reader must never observe a sid in + /// `output_to_storage_handles`. A concurrent reader must never observe a sid in /// `instances` that is missing from the secondary indexes (or vice /// versa). To preserve this, every writer ([`Self::register`], /// [`Self::remove_instance`]) acquires ALL THREE write guards - /// together in the fixed order `instances → policy_to_series_ids → + /// together in the fixed order `instances → output_to_storage_handles → /// metric_to_series_ids` BEFORE mutating any of them, so the update is /// atomic with respect to any reader that takes the `instances` lock. /// The fixed acquisition order is also the deadlock-avoidance order: /// no code path takes these locks in a different order. - policy_to_series_ids: RwLock>>, + output_to_storage_handles: RwLock>>, /// Secondary index: `metric_name → {sids}` (P2-2). Lets /// [`Self::instances_matching`] do a keyed lookup of the sids for a /// metric instead of an O(N) full scan of `instances`. Maintained in /// lock-step with `instances` under the same write-lock domain (see - /// the atomicity invariant on `policy_to_series_ids`). Holds every + /// the atomicity invariant on `output_to_storage_handles`). Holds every /// registered sid (UNSET-policy sids included), since the query path /// keys candidate selection on metric name, not policy. metric_to_series_ids: RwLock>>, - /// Pointer (as `usize`) of the `Arc` this store + /// Pointer (as `usize`) of the `Arc` this store /// last reconciled against. `reconcile_from_streaming_config` runs /// on every ingest batch, but the config is a lock-free - /// `Arc>` that only changes its `Arc` + /// `Arc>` that only changes its `Arc` /// identity on a control-plane swap (rare). Gating the full /// catalog scan on a cheap pointer compare against this field lets /// the steady-state ingest path skip reconcile entirely. @@ -696,7 +698,7 @@ pub struct SketchStore { /// has sealed epochs to persist) with retention-drop disabled (the /// flush-then-evict loop is the memory bound). persistence_read: RwLock>>, - persistence_metadata: RwLock>>, + persistence_metadata: RwLock>>, immutable_publisher: RwLock>, removed_sids: RwLock< BTreeMap< @@ -746,26 +748,73 @@ pub enum SeriesLookup { pub(crate) struct SummaryPublicationWriter<'a>(&'a SketchStore); impl SummaryPublicationWriter<'_> { - pub(crate) fn ingest_precompute_with_series_id( + pub(crate) fn write_stored_summary( &self, - sid: u64, + address: &asap_types::sds::StoredSummaryKey, + resolver: &crate::drivers::ingest::series_resolver::SeriesIdResolver, config: &asap_types::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { + address.validate().ok()?; + let expected = self + .0 + .descriptors + .stored_output_reference(address.stored_output_id)?; + if address.stored_output_id.fingerprint() != output.policy_fp + || output + .stored_output_reference + .as_ref() + .is_some_and(|reference| expected.validate().is_ok() && reference != &expected) + || config.policy_fingerprint() != output.policy_fp + { + return None; + } + let (population_key, population) = build_attrs_fp_and_label_map(config, output).ok()?; + if population != address.population + || (output.start_timestamp, output.end_timestamp) + != ( + u64::try_from(address.window.start_ms).ok()?, + u64::try_from(address.window.end_ms).ok()?, + ) + { + return None; + } + let generation = output.catalog_generation.as_deref(); + if generation + .is_some_and(|g| (g.plan_id, g.plan_version) != (address.plan_id, address.plan_version)) + { + return None; + } + let handle = self + .0 + .resolve_output_storage_handle( + resolver, + address.stored_output_id, + &population_key, + generation, + ) + .ok()?; + if output + .storage_handle + .is_some_and(|captured| captured != handle) + { + return None; + } self.0 - .ingest_precompute_with_admission(sid, config, output, state) + .ingest_precompute_with_admission(handle, config, output, state) } - pub(crate) fn ingest_precompute_for_agg_config>>( + #[cfg(test)] + pub(crate) fn ingest_precompute_with_series_id( &self, - mint: impl FnOnce(&str, &str, &str) -> R, + sid: u64, config: &asap_types::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { self.0 - .ingest_precompute_config_with_admission(mint, config, output, state) + .ingest_precompute_with_admission(sid, config, output, state) } #[cfg(test)] @@ -787,6 +836,34 @@ impl SketchStore { Self::default() } + /// Resolve a local row handle from the installed DAG output and population. + /// The durable identity includes the plan version and exact output reference. + pub fn resolve_output_storage_handle( + &self, + resolver: &crate::drivers::ingest::series_resolver::SeriesIdResolver, + definition: StoredOutputId, + population_key: &str, + generation: Option<&CatalogGeneration>, + ) -> Result { + self.validate_routed_catalog_generation(generation)?; + let output = self + .descriptors + .stored_output_reference(definition) + .ok_or("definition has no installed stored output")?; + let (plan_id, plan_version) = match generation { + Some(generation) => (generation.plan_id, generation.plan_version), + #[cfg(test)] + None => (0, 0), + #[cfg(not(test))] + None => return Err("stored output requires an installed plan generation".into()), + }; + let identity = + serde_json::to_string(&(plan_id, plan_version, output)).map_err(|e| e.to_string())?; + resolver + .try_resolve("stored-output", population_key, &identity) + .map_err(|e| e.to_string()) + } + /// Classify a sid for query routing. See `SeriesLookup` for semantics. pub fn classify(&self, sid: u64) -> SeriesLookup { let known = self.instances.read().unwrap().contains_key(&sid); @@ -814,11 +891,11 @@ impl SketchStore { /// ## Atomicity (P2-1) /// /// All three index write guards are acquired together, in the fixed - /// order `instances → policy_to_series_ids → metric_to_series_ids`, BEFORE any + /// order `instances → output_to_storage_handles → metric_to_series_ids`, BEFORE any /// map is mutated. This makes the three updates atomic with respect /// to a concurrent reader that takes the `instances` lock: such a /// reader can never see the sid in `instances` while it is still - /// absent from `policy_to_series_ids` / `metric_to_series_ids` (the pre-fix race + /// absent from `output_to_storage_handles` / `metric_to_series_ids` (the pre-fix race /// where the two indexes were written under separate sequential /// locks). See the index-field doc comments for the full invariant. pub fn register(&self, meta: SummarySeriesMetadata) { @@ -831,8 +908,7 @@ impl SketchStore { meta: SummarySeriesMetadata, instances: &mut HashMap, ) -> bool { - let sid = meta.sid; - let policy_fp = meta.policy_fp; + let sid = meta.storage_handle; // A non-legacy materialization must resolve through the installed // authoritative catalog. Unknown identities fail closed and never // become queryable SummaryStore entries. @@ -847,17 +923,20 @@ impl SketchStore { .data_descriptor .time_series_metric() .map(str::to_owned); - // Fixed lock order: instances → policy_to_series_ids → metric_to_series_ids. + // Fixed lock order: instances → output_to_storage_handles → metric_to_series_ids. if self.removed_sids.read().unwrap().contains_key(&sid) { tracing::warn!(sid, "rejecting reuse of a removed summary instance ID"); return false; } - let mut policy_idx = self.policy_to_series_ids.write().unwrap(); + let mut policy_idx = self.output_to_storage_handles.write().unwrap(); let mut metric_idx = self.metric_to_series_ids.write().unwrap(); - instances.insert(sid, instance); - if !policy_fp.is_unset() { - policy_idx.entry(policy_fp).or_default().insert(sid); + if let Some(output) = &instance.stored_output_reference { + policy_idx + .entry(output.stored_output_id) + .or_default() + .insert(sid); } + instances.insert(sid, instance); if let Some(metric_name) = metric_name { metric_idx.entry(metric_name).or_default().insert(sid); } @@ -866,10 +945,65 @@ impl SketchStore { /// Install one authoritative catalog snapshot for future registrations. /// Existing Series IDs retain their generation's descriptor Arcs while draining. + #[cfg(test)] pub fn install_summary_catalog( &self, catalog: Arc, ) -> Result<(), String> { + let outputs = catalog + .outputs + .keys() + .map(|id| (*id, catalog.output_reference(*id).unwrap())) + .collect(); + self.install_catalog_outputs(catalog, outputs) + } + + /// Install the exact selected writer references together with their catalog. + pub fn install_precompute_plan( + &self, + catalog: Arc, + plan: &asap_types::precompute_plan::PrecomputePlan, + ) -> Result<(), String> { + plan.validate_against_catalog(&catalog) + .map_err(|e| e.to_string())?; + let outputs = plan + .schemas + .iter() + .map(|schema| { + ( + schema.materialization, + schema.stored_output_reference.clone(), + ) + }) + .collect::>(); + if outputs.len() != plan.schemas.len() { + return Err("stored output bindings must be unique per definition".into()); + } + self.install_catalog_outputs(catalog, outputs) + } + + fn install_catalog_outputs( + &self, + catalog: Arc, + outputs: BTreeMap, + ) -> Result<(), String> { + for (definition, output) in &outputs { + output.validate().map_err(|e| e.to_string())?; + if output.stored_output_id != *definition + || catalog.output_reference(*definition).ok().as_ref() != Some(output) + { + return Err("stored output does not match the installed catalog".into()); + } + } + if outputs + .values() + .map(|output| output.stored_output_id) + .collect::>() + .len() + != outputs.len() + { + return Err("a stored output ID cannot name different definitions".into()); + } let reference = catalog.reference().map_err(|error| error.to_string())?; if let Some(writer) = self.persistence_metadata.read().unwrap().as_ref() { writer @@ -893,7 +1027,7 @@ impl SketchStore { .map_err(|error| error.to_string())? .flatten(); self.descriptors - .install_catalog(Arc::clone(&catalog)) + .install_catalog_with_outputs(Arc::clone(&catalog), outputs) .map_err(|error| error.to_string())?; inventory.install(generation.clone()); if closed.as_ref() == Some(&generation) { @@ -1129,11 +1263,48 @@ impl SketchStore { if policy_fp.is_unset() { return Vec::new(); } + let Some(output) = self.descriptors.stored_output_reference(policy_fp.into()) else { + return Vec::new(); + }; + self.storage_handles_for_output(&output) + } + + pub(crate) fn is_current_storage_handle(&self, handle: u64) -> bool { + let generation = self.active_catalog_generation(); + self.instances + .read() + .unwrap() + .get(&handle) + .is_some_and(|binding| { + binding.catalog_generation == generation + && binding.stored_output_reference + == self + .descriptors + .stored_output_reference(binding.policy_fp.into()) + }) + } + + pub fn stored_output_for_handle(&self, handle: u64) -> Option { + self.instances + .read() + .unwrap() + .get(&handle) + .and_then(|binding| binding.stored_output_reference.clone()) + } + + pub fn storage_handles_for_output(&self, output: &StoredOutputReference) -> Vec { + if self + .descriptors + .stored_output_reference(output.stored_output_id) + != Some(output.clone()) + { + return Vec::new(); + } let candidates: Vec<_> = self - .policy_to_series_ids + .output_to_storage_handles .read() .unwrap() - .get(&policy_fp) + .get(&output.stored_output_id) .map(|set| set.iter().copied().collect()) .unwrap_or_default(); let generation = self.active_catalog_generation(); @@ -1166,7 +1337,7 @@ impl SketchStore { /// least one sid. Useful for telemetry / `/runtime` introspection /// (mirrors the legacy "active aggregation count" metric). pub fn policy_count(&self) -> usize { - self.policy_to_series_ids.read().unwrap().len() + self.output_to_storage_handles.read().unwrap().len() } /// Look up the metadata for a sid (cloned because callers usually @@ -1263,17 +1434,30 @@ impl SketchStore { .map_err(|_| "summary instance start exceeds signed timestamp range")?; let end_ms = i64::try_from(window.1) .map_err(|_| "summary instance end exceeds signed timestamp range")?; - let group_fingerprint = xxhash_rust::xxh64::xxh64( - &serde_json::to_vec(&group_values).map_err(|error| error.to_string())?, - 0, - ); - let instance_id = asap_types::sds::SummaryInstanceCoordinates { + let output = StoredOutputReference { stored_output_id: *stored_output_id, - time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, - group_values: group_values.clone(), + definition_id: binding + .stored_output_reference + .as_ref() + .ok_or("missing semantic binding")? + .definition_id + .clone(), + }; + if binding.stored_output_reference != Some(output.clone()) { + return Err("inventory output differs from stored payload binding".into()); } - .instance_id() - .map_err(|error| error.to_string())?; + let address = asap_types::sds::StoredSummaryKey { + plan_id: generation.plan_id, + plan_version: generation.plan_version, + stored_output_id: output.stored_output_id, + population: group_values.clone(), + window: HalfOpenTimeRange { start_ms, end_ms }, + }; + let instance_id = asap_types::sds::SummaryInstanceId::new( + address.storage_key().map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let payload_address = address; let instance = SummaryInstance { instance_id: instance_id.clone(), stored_output_id: *stored_output_id, @@ -1294,10 +1478,7 @@ impl SketchStore { }, state_reference: SummaryStateReference { store: "summary-store".into(), - key: format!( - "series:{series_id}:pane:{}-{}:group:{group_fingerprint:016x}", - window.0, window.1 - ), + key: payload_address.storage_key().map_err(|e| e.to_string())?, state_schema_version: binding.summary_descriptor.state_schema_version, generation: generation.plan_version, sequence: window.1, @@ -2635,7 +2816,7 @@ impl SketchStore { } /// Record that the store has reconciled against the - /// `Arc` identified by `config_ptr` (the value of + /// `Arc` identified by `config_ptr` (the value of /// `Arc::as_ptr(..) as usize`), returning `true` if this is a *new* /// config pointer (i.e. the caller should run a full reconcile) or /// `false` if the store already reconciled against this exact @@ -2686,25 +2867,34 @@ impl SketchStore { .collect() } - fn metadata_record(&self, m: &SdsBinding) -> Option { + fn metadata_record( + &self, + m: &SdsBinding, + ) -> Option { let mut record = self.metadata_record_without_completion(m)?; - record.completed_through_ms = self.completed_windows.read().unwrap().get(&m.sid).copied(); + record.completed_through_ms = self + .completed_windows + .read() + .unwrap() + .get(&m.storage_handle) + .copied(); Some(record) } fn metadata_record_without_completion( &self, m: &SdsBinding, - ) -> Option { + ) -> Option { let mut record = - crate::storage_engines::sketch_db::index::persistence::metadata::SidMetaRecord::new( - m.sid, + crate::storage_engines::sketch_db::index::persistence::metadata::StoredOutputMetadataRecord::new( + m.storage_handle, m.metric_name.clone(), m.group_by_keys.iter().cloned().collect(), &m.agg_kind, m.first_seen_unix_ms, ); if !m.policy_fp.is_unset() { + record.stored_output_reference = m.stored_output_reference.clone(); record.stored_output_id = Some(StoredOutputId::from(m.policy_fp)); record.summary_definition_id = Some(m.stored_output_reference.as_ref()?.definition_id.clone()); @@ -2728,14 +2918,14 @@ impl SketchStore { .load() .map_err(|error| error.to_string())? .into_iter() - .find(|record| record.sid == instance.sid) + .find(|record| record.storage_handle == instance.storage_handle) .ok_or("cannot persist lifecycle without catalog identity")?, }; record.retired_at_ms = instance.retired_at_ms; record.expires_at_ms = instance.expires_at_ms; record.removed = removed; writer.upsert_all(&[record]).map_err(|error| { - tracing::error!(sid = instance.sid, %error, "durable summary lifecycle publication failed"); + tracing::error!(sid = instance.storage_handle, %error, "durable summary lifecycle publication failed"); error.to_string() }) } @@ -2862,7 +3052,7 @@ impl SketchStore { } /// Drop a sid's metadata + its series state + both secondary-index - /// entries (`policy_to_series_ids` and `metric_to_series_ids`). Mirrors + /// entries (`output_to_storage_handles` and `metric_to_series_ids`). Mirrors /// `SchemaRegistry::remove_schema` for the eviction path's /// post-data-drop cleanup. Returns the removed metadata, or `None` /// if the sid was absent. @@ -2870,7 +3060,7 @@ impl SketchStore { /// ## Atomicity (P2-1) /// /// Takes all three index write guards together in the fixed order - /// `instances → policy_to_series_ids → metric_to_series_ids` so the removal is + /// `instances → output_to_storage_handles → metric_to_series_ids` so the removal is /// atomic with respect to a concurrent reader — the sid never /// lingers in a secondary index after it has left `instances`. The /// `series` DashMap is touched after the index guards are released @@ -2879,7 +3069,7 @@ impl SketchStore { pub fn remove_instance(&self, sid: u64) -> Option> { let _mutation = self.begin_state_mutation(); let removed = { - // Fixed lock order: instances → policy_to_series_ids → metric_to_series_ids. + // Fixed lock order: instances → output_to_storage_handles → metric_to_series_ids. let mut instances = self.instances.write().ok()?; if let Some(instance) = instances.get(&sid) { self.persist_lifecycle(instance, true).ok()?; @@ -2894,15 +3084,15 @@ impl SketchStore { ), ); } - let mut policy_idx = self.policy_to_series_ids.write().unwrap(); + let mut policy_idx = self.output_to_storage_handles.write().unwrap(); let mut metric_idx = self.metric_to_series_ids.write().unwrap(); let removed = instances.remove(&sid); if let Some(meta) = &removed { - if !meta.policy_fp.is_unset() { - if let Some(set) = policy_idx.get_mut(&meta.policy_fp) { + if let Some(output) = &meta.stored_output_reference { + if let Some(set) = policy_idx.get_mut(&output.stored_output_id) { set.remove(&sid); if set.is_empty() { - policy_idx.remove(&meta.policy_fp); + policy_idx.remove(&output.stored_output_id); } } } @@ -2977,6 +3167,7 @@ impl SketchStore { /// once a sid is retired by [`crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config`] /// further writes are rejected here so the eviction sweep can /// drop residual state cleanly. + #[cfg(test)] pub fn ingest_precompute_for_agg_config>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, @@ -2991,6 +3182,7 @@ impl SketchStore { self.ingest_precompute_config_with_admission(mint_sid, agg_cfg, output, accumulator) } + #[cfg(test)] fn ingest_precompute_config_with_admission>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, @@ -3065,7 +3257,7 @@ impl SketchStore { // query path; capability-matching couldn't see it. if !self.register_with_instances( SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: agg_cfg.metric.clone(), group_by_keys, capability: Some(capability), @@ -3109,6 +3301,7 @@ impl SketchStore { /// The §6.3 ingest barrier (`Retired` / `Expired` sids reject /// writes) and first-sight metadata registration are identical to /// the mint-driven path. + #[cfg(test)] pub fn ingest_precompute_with_series_id( &self, sid: u64, @@ -3257,8 +3450,9 @@ impl SketchStore { // Publish the writer before recovery or any background work so a // concurrent lifecycle operation cannot succeed without persistence. - let metadata_writer = - Arc::new(persistence::metadata::SidMetadataStore::new(&cfg.disk_path)); + let metadata_writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( + &cfg.disk_path, + )); if let Some(catalog) = self.descriptors.authoritative_catalog() { metadata_writer.persist_catalog(&catalog)?; } @@ -3284,7 +3478,7 @@ impl SketchStore { .filter(|record| record.removed) .map(|record| { ( - record.sid, + record.storage_handle, (record.catalog_generation, record.stored_output_id), ) }), @@ -3362,8 +3556,8 @@ impl SketchStore { &self, disk_path: &std::path::Path, ) -> persistence::PersistResult { - use crate::storage_engines::sketch_db::index::persistence::metadata::SidMetadataStore; - let records = SidMetadataStore::new(disk_path).load()?; + use crate::storage_engines::sketch_db::index::persistence::metadata::StoredOutputMetadataFile; + let records = StoredOutputMetadataFile::new(disk_path).load()?; let mut registered = 0usize; for rec in records { @@ -3371,7 +3565,7 @@ impl SketchStore { self.completed_windows .write() .unwrap() - .entry(rec.sid) + .entry(rec.storage_handle) .and_modify(|current| *current = (*current).max(end)) .or_insert(end); } @@ -3379,15 +3573,16 @@ impl SketchStore { continue; } // Don't clobber a live-registered instance. - if self.instance(rec.sid).is_some() { + if self.instance(rec.storage_handle).is_some() { continue; } let catalog = self.descriptors.authoritative_snapshot(); let policy_fp = match (&rec.stored_output_id, &rec.catalog_generation, &catalog) { (Some(definition), Some(generation), Some((catalog, installed_generation))) => { - let persisted = persistence::metadata::SidMetadataStore::new(disk_path) + let persisted = persistence::metadata::StoredOutputMetadataFile::new(disk_path) .load_catalog(generation); if generation != installed_generation + || rec.stored_output_reference != catalog.output_reference(*definition).ok() || persisted.as_ref().ok() != Some(catalog.as_ref()) || !catalog.outputs.get(definition).is_some_and(|output| { Some(&output.definition_id) == rec.summary_definition_id.as_ref() @@ -3400,13 +3595,13 @@ impl SketchStore { && generation.plan_version < installed_generation.plan_version && catalog.outputs.contains_key(definition) { - self.removed_sids - .write() - .unwrap() - .insert(rec.sid, (Some(Arc::clone(generation)), Some(*definition))); + self.removed_sids.write().unwrap().insert( + rec.storage_handle, + (Some(Arc::clone(generation)), Some(*definition)), + ); } tracing::warn!( - sid = rec.sid, + sid = rec.storage_handle, "persisted summary catalog provenance differs; leaving state unbound" ); continue; @@ -3417,13 +3612,13 @@ impl SketchStore { // legacy path. Never promote such state into a catalog binding. (None, None, None) => PolicyFingerprint::UNSET, _ => { - tracing::warn!(sid = rec.sid, "persisted summary has no matching authoritative identity; leaving state unbound"); + tracing::warn!(sid = rec.storage_handle, "persisted summary has no matching authoritative identity; leaving state unbound"); continue; } }; let Some(agg_kind) = rec.agg_kind() else { tracing::warn!( - sid = rec.sid, + sid = rec.storage_handle, "skipping recovered sid: unrecognized agg_kind in sidecar" ); continue; @@ -3431,7 +3626,7 @@ impl SketchStore { let capability = rec.capability(); let accuracy = rec.accuracy(); self.register(SummarySeriesMetadata { - sid: rec.sid, + storage_handle: rec.storage_handle, metric_name: rec.metric_name, group_by_keys: rec.group_by_keys.into_iter().collect(), capability, @@ -3442,7 +3637,7 @@ impl SketchStore { expires_at_ms: rec.expires_at_ms, policy_fp, }); - if self.instance(rec.sid).is_some() { + if self.instance(rec.storage_handle).is_some() { registered += 1; } } @@ -3516,8 +3711,9 @@ impl crate::storage_engines::sketch_db::index::persistence::EpochSource for Sket fn instance_metadata_for_persist( &self, sid: u64, - ) -> Option - { + ) -> Option< + crate::storage_engines::sketch_db::index::persistence::metadata::StoredOutputMetadataRecord, + > { let g = self.instances.read().ok()?; let m = g.get(&sid)?; self.metadata_record(m) @@ -3722,7 +3918,7 @@ mod tests { relative_accuracy: 0.01, }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "m".into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), @@ -3775,7 +3971,7 @@ mod tests { fn recovery_rejects_unsupported_or_corrupt_metadata() { let directory = tempfile::tempdir().unwrap(); let store = SketchStore::new(); - let sidecar = persistence::metadata::SidMetadataStore::new(directory.path()); + let sidecar = persistence::metadata::StoredOutputMetadataFile::new(directory.path()); assert_eq!( store .register_recovered_disk_series(directory.path()) @@ -4337,11 +4533,11 @@ mod tests { let retired = idx.list_by_status(AggStatus::Retired); let expired = idx.list_by_status(AggStatus::Expired); assert_eq!(active.len(), 1); - assert_eq!(active[0].sid, 1); + assert_eq!(active[0].storage_handle, 1); assert_eq!(retired.len(), 1); - assert_eq!(retired[0].sid, 2); + assert_eq!(retired[0].storage_handle, 2); assert_eq!(expired.len(), 1); - assert_eq!(expired[0].sid, 3); + assert_eq!(expired[0].storage_handle, 3); } #[test] @@ -4351,7 +4547,7 @@ mod tests { idx.append_sample(1, BTreeMap::new(), (0, 10), sample(1)); assert_eq!(idx.classify(1), SeriesLookup::Hit); let removed = idx.remove_instance(1).expect("sid known"); - assert_eq!(removed.sid, 1); + assert_eq!(removed.storage_handle, 1); assert_eq!(idx.classify(1), SeriesLookup::Unknown); } @@ -5078,7 +5274,7 @@ mod tests { #[test] fn catalog_recovery_keeps_legacy_and_foreign_generation_state_unbound() { use crate::storage_engines::sketch_db::index::persistence::metadata::{ - SidMetaRecord, SidMetadataStore, + StoredOutputMetadataFile, StoredOutputMetadataRecord, }; let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( @@ -5102,15 +5298,15 @@ mod tests { .clone(); let fingerprint = state_config.policy_fingerprint(); let metadata = meta_for_config(507, &state_config); - let record = SidMetaRecord::new( - metadata.sid, + let record = StoredOutputMetadataRecord::new( + metadata.storage_handle, metadata.metric_name.clone(), metadata.group_by_keys.iter().cloned().collect(), &metadata.agg_kind, 0, ); let tmp = tempfile::tempdir().unwrap(); - let sidecar = SidMetadataStore::new(tmp.path()); + let sidecar = StoredOutputMetadataFile::new(tmp.path()); sidecar.upsert_all(&[record.clone()]).unwrap(); let store = SketchStore::new(); store @@ -5361,7 +5557,7 @@ mod tests { .unwrap(); store.register(meta_for_config(850, &state_config)); let generation = store.active_catalog_generation().unwrap(); - let writer = Arc::new(persistence::metadata::SidMetadataStore::new( + let writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( directory.path(), )); *store.persistence_metadata.write().unwrap() = Some(writer.clone()); @@ -5548,7 +5744,7 @@ mod tests { let store = SketchStore::new(); store.register(meta(800)); let directory = tempfile::tempdir().unwrap(); - let writer = Arc::new(persistence::metadata::SidMetadataStore::new( + let writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( directory.path(), )); // A directory in place of the sidecar causes the real writer to fail. @@ -5731,7 +5927,14 @@ mod tests { assert!(!inventory.instances.is_empty()); assert!(inventory.instances.values().all(|instance| { instance.group_values.get("job").map(String::as_str) == Some("api") - && instance.state_reference.key.starts_with("series:506:pane:") + && serde_json::from_str::( + &instance.state_reference.key, + ) + .is_ok_and(|key| { + key.stored_output_id.fingerprint() == fingerprint + && key.population == instance.group_values + && key.window == instance.time_range + }) })); } @@ -5761,7 +5964,7 @@ mod tests { fn meta_kll_host(sid: u64) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "http_latency".into(), group_by_keys: ["host".to_string()].into_iter().collect(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), @@ -6446,4 +6649,268 @@ mod tests { } persistence.shutdown(); } + // Exact selected-output bindings survive disk recovery and cannot be + // rebound to a different output merely because its definition matches. + #[test] + fn stored_output_reference_isolates_identity_and_survives_restart() { + use crate::drivers::ingest::series_resolver::SeriesIdResolver; + let snapshot = serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap(); + let config = plan + .precompute_plan + .materializations + .iter() + .find(|config| { + matches!( + config.accumulator_spec().unwrap().family, + planner_types::post_asap::SummaryFamilyType::Sketch(..) + ) + }) + .unwrap() + .clone(); + let output = plan + .summary_catalog + .output_reference(config.policy_fingerprint().into()) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let disk = directory.path().join("state"); + let wal = directory.path().join("outputs.wal"); + let handle; + { + let store = Arc::new(SketchStore::new()); + store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan, + ) + .unwrap(); + let resolver = SeriesIdResolver::open(wal.clone()).unwrap(); + let generation = store.active_catalog_generation().unwrap(); + handle = store + .resolve_output_storage_handle( + &resolver, + output.stored_output_id, + "population-a", + Some(&generation), + ) + .unwrap(); + let other_population = store + .resolve_output_storage_handle( + &resolver, + output.stored_output_id, + "population-b", + Some(&generation), + ) + .unwrap(); + assert_ne!(handle, other_population); + store.register(meta_for_config(handle, &config)); + let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); + for pane in 0..4 { + store.append_sample( + handle, + BTreeMap::new(), + (pane * 30_000, (pane + 1) * 30_000), + sample(1), + ); + } + assert!(wait_until( + || !persistence.manifest.live_parts().is_empty() + && store.list_sealed_epochs_len() == 0, + Duration::from_secs(5) + )); + assert_eq!(store.storage_handles_for_output(&output), vec![handle]); + let wrong = StoredOutputReference { + stored_output_id: asap_types::sds::StoredOutputId(102), + ..output.clone() + }; + assert!(store.storage_handles_for_output(&wrong).is_empty()); + persistence.shutdown(); + } + { + let store = Arc::new(SketchStore::new()); + store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan, + ) + .unwrap(); + let resolver = SeriesIdResolver::open(wal.clone()).unwrap(); + let generation = store.active_catalog_generation().unwrap(); + assert_eq!( + store + .resolve_output_storage_handle( + &resolver, + output.stored_output_id, + "population-a", + Some(&generation) + ) + .unwrap(), + handle + ); + let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); + assert_eq!(store.storage_handles_for_output(&output), vec![handle]); + assert!(!store.query_range(handle, 0, 120_000).is_empty()); + persistence.shutdown(); + } + plan.precompute_plan.schemas[0] + .stored_output_reference + .stored_output_id = asap_types::sds::StoredOutputId(102); + let store = SketchStore::new(); + assert!(store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan + ) + .is_err()); + } + // A versioned output address cannot silently resolve the previous version. + #[test] + fn stored_output_reference_does_not_alias_previous_plan_version() { + let snapshot = serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap(); + let output = plan.precompute_plan.schemas[0] + .stored_output_reference + .clone(); + let store = SketchStore::new(); + store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan, + ) + .unwrap(); + store.register(meta_for_config( + 100, + plan.precompute_plan + .materializations + .iter() + .find(|config| config.policy_fingerprint() == output.stored_output_id.fingerprint()) + .unwrap(), + )); + assert_eq!(store.storage_handles_for_output(&output), vec![100]); + let mut next = plan.summary_catalog; + next.plan_version += 1; + store + .install_catalog_outputs( + Arc::new(next), + plan.precompute_plan + .schemas + .iter() + .map(|schema| { + ( + schema.materialization, + schema.stored_output_reference.clone(), + ) + }) + .collect(), + ) + .unwrap(); + assert!(store.storage_handles_for_output(&output).is_empty()); + assert!(!store.is_current_storage_handle(100)); + } +} + +impl SketchStore { + /// Build an isolated read view from one already pinned query-wide snapshot. + pub(crate) fn from_revision( + plan: &crate::storage_engines::types::RuntimePhysicalPlan, + pinned: &crate::precompute_engine::revisions::PinnedRevision, + ) -> Result { + use crate::storage_engines::types::{KeyByLabelValues, PrecomputedOutput}; + let view = Self::new(); + view.install_precompute_plan( + plan.summary_catalog + .clone() + .ok_or("revision view requires catalog")?, + &plan.precompute_plan, + )?; + let generation = Arc::new( + plan.precompute_plan + .summary_catalog + .clone() + .ok_or("revision view requires generation")?, + ); + let mut sids = BTreeMap::new(); + for record in &pinned.records { + let definition = record.reference.stored_output_id; + let expected = plan + .installed_precompute_plan + .stored_output_reference(definition) + .ok_or("revision view has unbound output")?; + if expected != record.reference { + return Err("revision view semantic binding mismatch".into()); + } + let config = plan + .precompute_plan + .materializations + .iter() + .find(|c| c.policy_fingerprint() == definition.fingerprint()) + .ok_or("revision view lacks output configuration")?; + let next_sid = sids.len() as u64 + 1; + let sid = *sids + .entry((definition, record.group.clone())) + .or_insert(next_sid); + let mut output = PrecomputedOutput::new( + record.start_ms, + record.end_ms, + Some(KeyByLabelValues::new_with_labels( + config + .grouping_labels + .iter() + .map(|k| record.group.get(k).cloned().unwrap_or_default()) + .collect(), + )), + definition.fingerprint(), + ); + output.population_labels = Some(record.group.clone()); + output.catalog_generation = Some(Arc::clone(&generation)); + output.stored_output_reference = Some(record.reference.clone()); + let state = crate::precompute_engine::revisions::decode_state(record, config)?; + let labels = view + .register_precompute_output(sid, config, &output) + .ok_or("revision view rejected output metadata")?; + // This is an immutable read view of already committed revisions, + // not additive producer admission (which correctly rejects derived + // writes). Every payload was decoded against its installed family. + let accepted = + match crate::storage_engines::sketch_db::data::agg_kind_for_config(config) { + AggKind::Sketch { .. } => view.append_sample_with_binding( + sid, + labels, + (record.start_ms, record.end_ms), + SketchSampleState { + bytes: state.serialize_to_bytes(), + encoding: SketchEncoding::MsgpackFull, + }, + ), + AggKind::ExactAgg { .. } => view.append_precompute_with_binding( + sid, + labels, + (record.start_ms, record.end_ms), + state.clone_boxed_core(), + ), + }; + if !accepted { + return Err("revision view rejected committed payload".into()); + } + } + let mut samples = + crate::drivers::ingest::prometheus_remote_write::revision_samples(&pinned.inputs)?; + samples + .sort_by(|a, b| (a.timestamp_ms, &a.series_key).cmp(&(b.timestamp_ms, &b.series_key))); + view.current_series + .lock() + .map_err(|_| "revision current-series view poisoned")? + .ingest(&plan.query_plan, &samples); + Ok(view) + } } diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index c988369ab..b3edf16f2 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -115,7 +115,7 @@ impl SchemaEvictionService { let _ = &self.backfill; for meta in expired { - let sid = meta.sid; + let sid = meta.storage_handle; let metric = meta.metric_name.clone(); if self.config.dry_run { @@ -193,7 +193,7 @@ pub fn warn_if_retention_inverted( #[cfg(test)] mod tests { use super::*; - use crate::storage_engines::types::{AggregationType, StreamingConfig}; + use crate::storage_engines::types::{AggregationType, InstalledPrecomputePlan}; use asap_physical_operators::summary_kernels::SumAccumulator; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; @@ -235,7 +235,7 @@ mod tests { /// derived from each dummy_agg. Returns the config plus the /// marker_id → fingerprint mapping so callers can look up the /// right key. - fn make_streaming_config(ids: &[u64]) -> (Arc, HashMap) { + fn make_streaming_config(ids: &[u64]) -> (Arc, HashMap) { let mut map = HashMap::new(); let mut id_to_fp = HashMap::new(); for &id in ids { @@ -244,12 +244,12 @@ mod tests { id_to_fp.insert(id, fp); map.insert(fp, cfg); } - (Arc::new(StreamingConfig::new(map)), id_to_fp) + (Arc::new(InstalledPrecomputePlan::new(map)), id_to_fp) } fn write_one( summary_store: &SketchStore, - streaming_config: &StreamingConfig, + installed_precompute_plan: &InstalledPrecomputePlan, agg_id: u64, ts: u64, ) -> u64 { @@ -262,7 +262,9 @@ mod tests { None, asap_types::PolicyFingerprint(agg_id), ); - let agg_cfg = streaming_config.get_aggregation_config(agg_id).unwrap(); + let agg_cfg = installed_precompute_plan + .get_aggregation_config(agg_id) + .unwrap(); // Test-scoped resolver — each call mints fresh. Production // shares one resolver across all sinks; tests don't need that // because each fixture is isolated. Static-lifetime so multiple diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index 0b712f456..c5bd2d504 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -10,7 +10,7 @@ use std::collections::{BTreeSet, HashSet}; use std::sync::Arc; use std::time::Duration; -use crate::storage_engines::types::StreamingConfig; +use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; @@ -22,7 +22,7 @@ use crate::storage_engines::sketch_db::lifecycle::AggStatus; pub struct SidReconcileSummary { /// Sids that transitioned `Active → Retired` because their /// content signature is no longer present in the new - /// `StreamingConfig`. Already-Retired or already-Expired sids + /// `InstalledPrecomputePlan`. Already-Retired or already-Expired sids /// are not re-touched. pub retired: Vec, } @@ -35,14 +35,14 @@ pub struct SidReconcileSummary { /// behavior. /// Ingest-path entry point: reconcile only when `config` is a config /// the store has not yet reconciled against. The streaming config is a -/// lock-free `Arc>` whose `Arc` identity only +/// lock-free `Arc>` whose `Arc` identity only /// changes on a (rare) control-plane swap, so in steady state every /// ingest batch hands us the *same* `Arc`. Gating on the `Arc` data /// pointer collapses the per-batch reconcile to a single relaxed atomic /// load in that common case, skipping the full catalog scan + any /// signature derivation. /// -/// Pass the same `Arc` the ingest batch snapshotted so +/// Pass the same `Arc` the ingest batch snapshotted so /// the pointer is stable; the snapshot is held alive for the call's /// duration, so the pointer cannot be reused by a concurrently-dropped /// config (no ABA hazard within a batch). @@ -51,7 +51,7 @@ pub struct SidReconcileSummary { /// `Some(summary)` with the retired sids when a reconcile actually ran. pub fn reconcile_if_config_changed( store: &SketchStore, - config: &Arc, + config: &Arc, retention: Duration, ) -> Option { let config_ptr = Arc::as_ptr(config) as usize; @@ -63,7 +63,7 @@ pub fn reconcile_if_config_changed( pub fn reconcile_from_streaming_config( store: &SketchStore, - config: &StreamingConfig, + config: &InstalledPrecomputePlan, retention: Duration, ) -> SidReconcileSummary { let live_signatures = build_live_signature_set(config); @@ -176,7 +176,7 @@ fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { signature_bytes(&cfg.metric, &agg_kind, &group_by_keys) } -fn build_live_signature_set(config: &StreamingConfig) -> HashSet> { +fn build_live_signature_set(config: &InstalledPrecomputePlan) -> HashSet> { config .materializations() .values() @@ -304,7 +304,7 @@ mod tests { ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys, capability: None, @@ -321,12 +321,12 @@ mod tests { } } - fn streaming(configs: Vec) -> StreamingConfig { + fn streaming(configs: Vec) -> InstalledPrecomputePlan { let mut map = HashMap::new(); for (i, c) in configs.into_iter().enumerate() { map.insert(i as u64 + 1, c); } - StreamingConfig::new(map) + InstalledPrecomputePlan::new(map) } #[test] @@ -410,7 +410,7 @@ mod tests { ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys, capability: None, diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs index 78f5e8df9..f73f1a69b 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs @@ -13,11 +13,11 @@ use std::time::Duration; /// clock crosses `expires_at_ms`, without any state mutation). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum AggStatus { - /// Sid is reachable from the current `StreamingConfig`. Writes + /// Sid is reachable from the current `InstalledPrecomputePlan`. Writes /// accepted; queries see live data. Active, /// Sid's signature no longer appears in the current - /// `StreamingConfig` but its data is still within retention. + /// `InstalledPrecomputePlan` but its data is still within retention. /// Writes rejected by the §6.3 sid-level ingest barrier /// (`SketchStore::ingest_precompute_for_agg_config` returns /// `None`); reads allowed for queries that reference the diff --git a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs index 6018bdeb9..dc9e8cb6c 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs @@ -42,7 +42,7 @@ pub(crate) struct FlusherShared { /// Per-sid metadata sidecar — upserted on every flush so recovery can /// re-register disk-resident sids as queryable instances. See /// [`super::metadata`]. - pub sid_metadata: Arc, + pub sid_metadata: Arc, pub next_part_id: AtomicU64, pub shutdown: AtomicBool, /// Woken by the insert path when it hits `hard_cap_bytes` and by @@ -77,7 +77,9 @@ impl FlusherHandle { where S: EpochSource + 'static, { - let metadata = Arc::new(super::metadata::SidMetadataStore::new(&cfg.disk_path)); + let metadata = Arc::new(super::metadata::StoredOutputMetadataFile::new( + &cfg.disk_path, + )); Self::start_with_metadata(cfg, manifest, source, metadata) } @@ -85,7 +87,7 @@ impl FlusherHandle { cfg: SketchStorePersistenceConfig, manifest: Arc, source: Arc, - sid_metadata: Arc, + sid_metadata: Arc, ) -> PersistResult where S: EpochSource + 'static, @@ -134,7 +136,7 @@ impl FlusherHandle { }) } - pub(crate) fn metadata_store(&self) -> Arc { + pub(crate) fn metadata_store(&self) -> Arc { Arc::clone(&self.inner.sid_metadata) } @@ -345,7 +347,7 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR // is durable whenever the part it describes is. A sidecar write // failure must NOT abort the flush (the part is already durable) // — log and continue; recovery degrades to live-ingest re-register. - let sid_meta: Vec = { + let sid_meta: Vec = { let mut seen = std::collections::HashSet::new(); snapshots .iter() diff --git a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs index 4ea55e319..76025600b 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs @@ -5,7 +5,7 @@ use sha2::{Digest, Sha256}; use super::flusher::{FlusherHandle, FlusherShared}; use super::manifest::PartEntry; -use super::metadata::SidMetaRecord; +use super::metadata::StoredOutputMetadataRecord; use super::part::{part_dir_path, PartReader, PartWriter}; use super::source::{EpochSnapshot, EpochSnapshotEntry}; use super::{PersistError, PersistResult}; @@ -29,12 +29,15 @@ fn invalid(message: &str) -> PersistError { PersistError::Format(message.into()) } -fn validate_identity(current: &SidMetaRecord, record: &SidMetaRecord) -> PersistResult<()> { - if current.sid != record.sid +fn validate_identity( + current: &StoredOutputMetadataRecord, + record: &StoredOutputMetadataRecord, +) -> PersistResult<()> { + if current.storage_handle != record.storage_handle || current.removed || current.retired_at_ms.is_some() || current.expires_at_ms.is_some() - || current.stored_output_id != record.stored_output_id + || current.stored_output_reference != record.stored_output_reference || current.summary_definition_id != record.summary_definition_id || current.catalog_generation != record.catalog_generation || current.metric_name != record.metric_name @@ -101,7 +104,7 @@ impl FlusherHandle { } pub fn publish_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], snapshot: &EpochSnapshot, ) -> PersistResult { @@ -121,13 +124,13 @@ impl FlusherShared { /// randomized sketch again. This never creates or replaces a payload. pub fn lookup_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], start_ms: u64, end_ms: u64, ) -> PersistResult> { self.sid_metadata.transaction(|records, _| { - let Some(current) = records.get(&record.sid.to_string()) else { + let Some(current) = records.get(&record.storage_handle.to_string()) else { return Ok(None); }; validate_identity(current, record)?; @@ -140,7 +143,7 @@ impl FlusherShared { if previous.input_digest != input_digest { return Err(invalid("immutable lookup input lineage differs")); } - self.validate_reserved_part(record.sid, previous)?; + self.validate_reserved_part(record.storage_handle, previous)?; if !self .manifest .live_parts() @@ -160,16 +163,16 @@ impl FlusherShared { /// input and payload. This does not guarantee source availability after GC. pub fn publish_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], snapshot: &EpochSnapshot, ) -> PersistResult { - if snapshot.agg_id != record.sid || record.removed { + if snapshot.agg_id != record.storage_handle || record.removed { return Err(invalid("immutable publication SID is invalid or removed")); } let payload_digest = fingerprint(snapshot)?; self.sid_metadata.transaction(|records, metadata| { - let key = record.sid.to_string(); + let key = record.storage_handle.to_string(); let mut current = records.get(&key).cloned().unwrap_or_else(|| record.clone()); validate_identity(¤t, record)?; // Completion fences window ends. An advancing overlapping full @@ -192,7 +195,7 @@ impl FlusherShared { "completed immutable retry differs from latest publication", )); } - self.validate_reserved_part(record.sid, previous)?; + self.validate_reserved_part(record.storage_handle, previous)?; if !self .manifest .live_parts() @@ -237,7 +240,7 @@ impl FlusherShared { if path.exists() { // Never overwrite a reserved part silently: callers can recover a // durable part without sources; damaged/partial parts fail closed. - self.validate_reserved_part(record.sid, &reservation)?; + self.validate_reserved_part(record.storage_handle, &reservation)?; } else { PartWriter::write_part(&path, reservation.part_id, std::slice::from_ref(snapshot))?; std::fs::File::open( @@ -245,7 +248,7 @@ impl FlusherShared { .ok_or_else(|| invalid("missing parts parent"))?, )? .sync_all()?; - self.validate_reserved_part(record.sid, &reservation)?; + self.validate_reserved_part(record.storage_handle, &reservation)?; } self.finish_reserved_part(&mut current, &reservation)?; records.insert(key, current); @@ -261,12 +264,15 @@ impl FlusherShared { /// validation. A different pending input cannot be acknowledged by this call. pub fn resume_matching_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], start_ms: u64, end_ms: u64, ) -> PersistResult> { - self.resume_immutable_window(record.sid, Some((record, input_digest, start_ms, end_ms))) + self.resume_immutable_window( + record.storage_handle, + Some((record, input_digest, start_ms, end_ms)), + ) } /// Complete a pending durable part without reconstructing its source input. @@ -282,7 +288,7 @@ impl FlusherShared { fn resume_immutable_window( &self, sid: u64, - expected: Option<(&SidMetaRecord, [u8; 32], u64, u64)>, + expected: Option<(&StoredOutputMetadataRecord, [u8; 32], u64, u64)>, ) -> PersistResult> { self.sid_metadata.transaction(|records, metadata| { let key = sid.to_string(); @@ -359,7 +365,7 @@ impl FlusherShared { fn finish_reserved_part( &self, - current: &mut SidMetaRecord, + current: &mut StoredOutputMetadataRecord, pending: &ImmutableOutputReservation, ) -> PersistResult<()> { let path = part_dir_path(&self.cfg.disk_path.join("parts"), pending.part_id); @@ -435,8 +441,8 @@ mod tests { ) .unwrap() } - fn record() -> SidMetaRecord { - SidMetaRecord::new( + fn record() -> StoredOutputMetadataRecord { + StoredOutputMetadataRecord::new( 1, "derived".into(), vec![], diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index e6d8faa24..99df5ed24 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -300,8 +300,10 @@ impl AggKindRec { /// `capability` and `accuracy` are DERIVED from `agg_kind` on load, the /// same way the ingest path derives them, so the record stays minimal. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct SidMetaRecord { - pub sid: u64, +pub struct StoredOutputMetadataRecord { + #[serde(default)] + pub stored_output_reference: Option, + pub storage_handle: u64, /// Authoritative identity and provenance, absent on legacy sidecars. #[serde(default)] pub stored_output_id: Option, @@ -330,7 +332,7 @@ pub struct SidMetaRecord { pub last_immutable: Option, } -impl SidMetaRecord { +impl StoredOutputMetadataRecord { /// Build a record from the live store-side fields. `agg_kind` is the /// structured `AggKind`; `capability`/`accuracy` are intentionally /// NOT stored (re-derived on load). @@ -342,7 +344,8 @@ impl SidMetaRecord { first_seen_unix_ms: i64, ) -> Self { Self { - sid, + storage_handle: sid, + stored_output_reference: None, stored_output_id: None, summary_definition_id: None, catalog_generation: None, @@ -412,8 +415,10 @@ struct DataDescriptorRec { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -struct SidBindingRec { - sid: u64, +struct StoredOutputBindingRecord { + #[serde(default)] + stored_output_reference: Option, + storage_handle: u64, #[serde(default)] stored_output_id: Option, #[serde(default)] @@ -437,7 +442,7 @@ struct SidBindingRec { last_immutable: Option, } -/// Version-4 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold +/// Version-5 normalized sidecar with authoritative catalog provenance. Descriptors appear once and SeriesId bindings hold /// foreign keys, mirroring the in-memory SDS registry. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct SdsSidecar { @@ -446,15 +451,15 @@ struct SdsSidecar { catalog_generations: HashMap>, summary_descriptors: HashMap, data_descriptors: HashMap, - bindings: HashMap, + bindings: HashMap, } impl SdsSidecar { - fn from_records(records: impl IntoIterator) -> Self { + fn from_records(records: impl IntoIterator) -> Self { use crate::storage_engines::sketch_db::sds::{data_descriptor_id, summary_descriptor_id}; let mut sidecar = Self { - schema_version: 4, + schema_version: 5, catalog_generations: HashMap::new(), summary_descriptors: HashMap::new(), data_descriptors: HashMap::new(), @@ -494,9 +499,10 @@ impl SdsSidecar { digest }); sidecar.bindings.insert( - record.sid.to_string(), - SidBindingRec { - sid: record.sid, + record.storage_handle.to_string(), + StoredOutputBindingRecord { + storage_handle: record.storage_handle, + stored_output_reference: record.stored_output_reference, stored_output_id: record.stored_output_id, summary_definition_id: record.summary_definition_id, catalog_generation_sha256: generation_sha256, @@ -515,7 +521,7 @@ impl SdsSidecar { sidecar } - fn into_records(self) -> PersistResult> { + fn into_records(self) -> PersistResult> { self.bindings .into_values() .map(|binding| { @@ -525,7 +531,7 @@ impl SdsSidecar { .ok_or_else(|| { PersistError::Format(format!( "SeriesId {} references missing summary descriptor {}", - binding.sid, binding.summary_descriptor_id + binding.storage_handle, binding.summary_descriptor_id )) })?; let data = self @@ -534,11 +540,12 @@ impl SdsSidecar { .ok_or_else(|| { PersistError::Format(format!( "SeriesId {} references missing data descriptor {}", - binding.sid, binding.data_descriptor_id + binding.storage_handle, binding.data_descriptor_id )) })?; - Ok(SidMetaRecord { - sid: binding.sid, + Ok(StoredOutputMetadataRecord { + storage_handle: binding.storage_handle, + stored_output_reference: binding.stored_output_reference.clone(), stored_output_id: binding.stored_output_id, summary_definition_id: binding.summary_definition_id, catalog_generation: binding @@ -551,7 +558,7 @@ impl SdsSidecar { .ok_or_else(|| { PersistError::Format(format!( "SeriesId {} references missing catalog generation", - binding.sid + binding.storage_handle )) }) }) @@ -577,12 +584,12 @@ impl SdsSidecar { /// rewrite per flush tick is cheap and keeps the on-disk file always /// consistent with no log-replay machinery. #[derive(Debug)] -pub struct SidMetadataStore { +pub struct StoredOutputMetadataFile { path: PathBuf, writer: std::sync::Mutex<()>, } -impl SidMetadataStore { +impl StoredOutputMetadataFile { /// Open (or lazily create on first write) the sidecar at /// `/sid_metadata.json`. pub fn new(disk_path: &Path) -> Self { @@ -689,28 +696,28 @@ impl SidMetadataStore { /// Load current-format durable records. A missing file denotes a fresh store; /// malformed or unsupported persisted metadata is an error. - pub fn load(&self) -> PersistResult> { + pub fn load(&self) -> PersistResult> { self.load_strict() } /// Upsert a batch of records, merging with whatever is already on /// disk (last write wins per sid). Atomic via tmp + rename + dir /// fsync, matching the manifest's durability discipline. - pub fn upsert_all(&self, records: &[SidMetaRecord]) -> PersistResult<()> { + pub fn upsert_all(&self, records: &[StoredOutputMetadataRecord]) -> PersistResult<()> { let _writer = self.writer.lock().map_err(|_| { PersistError::Io(std::io::Error::other("summary metadata writer poisoned")) })?; if records.is_empty() { return Ok(()); } - let mut map: HashMap = self + let mut map: HashMap = self .load_strict()? .into_iter() - .map(|r| (r.sid.to_string(), r)) + .map(|r| (r.storage_handle.to_string(), r)) .collect(); let mut changed = false; for r in records { - let key = r.sid.to_string(); + let key = r.storage_handle.to_string(); let mut next = r.clone(); if let Some(existing) = map.get(&key) { // Lifecycle is monotone for a SeriesId. An older flush snapshot @@ -740,7 +747,7 @@ impl SidMetadataStore { self.write_atomic(json.as_bytes()) } - pub fn load_strict(&self) -> PersistResult> { + pub fn load_strict(&self) -> PersistResult> { let bytes = match fs::read(&self.path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), @@ -751,7 +758,7 @@ impl SidMetadataStore { if value .get("schema_version") .and_then(serde_json::Value::as_u64) - != Some(4) + != Some(5) { return Err(PersistError::Format( "unsupported SID metadata version".into(), @@ -764,7 +771,10 @@ impl SidMetadataStore { pub(super) fn transaction( &self, - operation: impl FnOnce(&mut HashMap, &Self) -> PersistResult, + operation: impl FnOnce( + &mut HashMap, + &Self, + ) -> PersistResult, ) -> PersistResult { let _writer = self .writer @@ -773,13 +783,13 @@ impl SidMetadataStore { let mut records = self .load_strict()? .into_iter() - .map(|r| (r.sid.to_string(), r)) + .map(|r| (r.storage_handle.to_string(), r)) .collect(); operation(&mut records, self) } pub(super) fn write_records( &self, - records: &HashMap, + records: &HashMap, ) -> PersistResult<()> { let sidecar = SdsSidecar::from_records(records.values().cloned()); let bytes = @@ -818,8 +828,8 @@ mod tests { use super::*; use tempfile::TempDir; - fn sketch_meta(sid: u64) -> SidMetaRecord { - SidMetaRecord::new( + fn sketch_meta(sid: u64) -> StoredOutputMetadataRecord { + StoredOutputMetadataRecord::new( sid, "http_latency".into(), vec!["host".into(), "zone".into()], @@ -832,8 +842,8 @@ mod tests { ) } - fn exact_meta(sid: u64) -> SidMetaRecord { - SidMetaRecord::new( + fn exact_meta(sid: u64) -> StoredOutputMetadataRecord { + StoredOutputMetadataRecord::new( sid, "http_requests_total".into(), vec!["zone".into()], @@ -849,7 +859,7 @@ mod tests { #[test] fn load_on_missing_file_is_empty() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); assert!(s.load().unwrap().is_empty()); } @@ -857,7 +867,7 @@ mod tests { #[test] fn persisted_definitions_roundtrip_and_reject_tampering() { let directory = tempfile::tempdir().unwrap(); - let store = SidMetadataStore::new(directory.path()); + let store = StoredOutputMetadataFile::new(directory.path()); let catalog = asap_types::summary_catalog::SummaryCatalog::build(7, 2, []).unwrap(); store.persist_catalog(&catalog).unwrap(); let generation = catalog.reference().unwrap(); @@ -876,7 +886,7 @@ mod tests { #[test] fn authoritative_bindings_share_one_persisted_catalog_generation() { let directory = tempfile::tempdir().unwrap(); - let store = SidMetadataStore::new(directory.path()); + let store = StoredOutputMetadataFile::new(directory.path()); let generation = std::sync::Arc::new(asap_types::sds::CatalogGeneration { schema_version: 1, plan_id: 7, @@ -887,7 +897,7 @@ mod tests { first.stored_output_id = Some(asap_types::PolicyFingerprint(7).into()); first.catalog_generation = Some(std::sync::Arc::clone(&generation)); let mut second = first.clone(); - second.sid = 2; + second.storage_handle = 2; store.upsert_all(&[first, second]).unwrap(); let json: serde_json::Value = serde_json::from_slice(&std::fs::read(store.path()).unwrap()).unwrap(); @@ -907,18 +917,18 @@ mod tests { #[test] fn upsert_then_load_round_trips() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); s.upsert_all(&[sketch_meta(1), exact_meta(2)]).unwrap(); let mut got = s.load().unwrap(); - got.sort_by_key(|r| r.sid); + got.sort_by_key(|r| r.storage_handle); assert_eq!(got.len(), 2); assert_eq!(got[0], sketch_meta(1)); assert_eq!(got[1], exact_meta(2)); let persisted: serde_json::Value = serde_json::from_slice(&std::fs::read(s.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 4); + assert_eq!(persisted["schema_version"], 5); assert_eq!( persisted["summary_descriptors"].as_object().unwrap().len(), 2 @@ -930,7 +940,7 @@ mod tests { #[test] fn equivalent_sids_persist_one_copy_of_each_descriptor() { let tmp = TempDir::new().unwrap(); - let store = SidMetadataStore::new(tmp.path()); + let store = StoredOutputMetadataFile::new(tmp.path()); let mut second = sketch_meta(2); second.first_seen_unix_ms = 9999; store.upsert_all(&[sketch_meta(1), second]).unwrap(); @@ -949,7 +959,7 @@ mod tests { #[test] fn broken_descriptor_reference_does_not_partially_recover() { let tmp = TempDir::new().unwrap(); - let store = SidMetadataStore::new(tmp.path()); + let store = StoredOutputMetadataFile::new(tmp.path()); store.upsert_all(&[sketch_meta(1), exact_meta(2)]).unwrap(); let mut persisted: serde_json::Value = @@ -970,10 +980,10 @@ mod tests { #[test] fn unsupported_sidecars_are_rejected_without_overwriting() { let tmp = TempDir::new().unwrap(); - let store = SidMetadataStore::new(tmp.path()); + let store = StoredOutputMetadataFile::new(tmp.path()); let flat = serde_json::to_value(HashMap::from([("1", sketch_meta(1))])).unwrap(); let mut cases = vec![flat]; - for version in [1, 2, 3, 5, 999] { + for version in [1, 2, 3, 4, 6, 999] { cases.push(serde_json::json!({"schema_version": version})); } for value in cases { @@ -988,7 +998,7 @@ mod tests { #[test] fn upsert_merges_and_overwrites_per_sid() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); s.upsert_all(&[sketch_meta(1)]).unwrap(); // New sid + updated metric for sid 1. let mut updated = sketch_meta(1); @@ -996,7 +1006,7 @@ mod tests { s.upsert_all(&[updated.clone(), exact_meta(2)]).unwrap(); let mut got = s.load().unwrap(); - got.sort_by_key(|r| r.sid); + got.sort_by_key(|r| r.storage_handle); assert_eq!(got.len(), 2); assert_eq!(got[0].metric_name, "http_latency_v2"); assert_eq!(got[1], exact_meta(2)); @@ -1022,7 +1032,7 @@ mod tests { #[test] fn unparsable_file_is_rejected() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); std::fs::write(s.path(), b"{not json").unwrap(); assert!(s.load().is_err()); } diff --git a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs index 552fe4e03..9627f08d2 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs @@ -32,7 +32,7 @@ pub mod recovery; pub use config::SketchStorePersistenceConfig; pub use manifest::{Manifest, PartEntry}; -pub use metadata::{SidMetaRecord, SidMetadataStore}; +pub use metadata::{StoredOutputMetadataFile, StoredOutputMetadataRecord}; pub use part::{PartId, PartReader, PartWriter, SnapshotEntry}; pub use source::{EpochSource, SealedEpochRef}; diff --git a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs index a21402608..ebf09525c 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs @@ -34,7 +34,7 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { // Validate every live part by reading its meta.bin header. let mut to_drop: Vec = Vec::new(); - let pending: HashSet = super::metadata::SidMetadataStore::new(disk_path) + let pending: HashSet = super::metadata::StoredOutputMetadataFile::new(disk_path) .load_strict()? .into_iter() .filter_map(|record| record.pending_immutable.map(|pending| pending.part_id)) diff --git a/data_plane/src/storage_engines/sketch_db/persistence/source.rs b/data_plane/src/storage_engines/sketch_db/persistence/source.rs index 661b6a927..6a384fb0d 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/source.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/source.rs @@ -142,7 +142,10 @@ pub trait EpochSource: Send + Sync { /// Returns `None` when the sid is unknown to the source (e.g. a test /// fake, or a sid whose instance metadata was evicted). Default impl /// returns `None` so existing/test sources need not implement it. - fn instance_metadata_for_persist(&self, _sid: u64) -> Option { + fn instance_metadata_for_persist( + &self, + _sid: u64, + ) -> Option { None } } diff --git a/data_plane/src/storage_engines/sketch_db/query/timeline.rs b/data_plane/src/storage_engines/sketch_db/query/timeline.rs index 92a64b0bf..0c8c4d035 100644 --- a/data_plane/src/storage_engines/sketch_db/query/timeline.rs +++ b/data_plane/src/storage_engines/sketch_db/query/timeline.rs @@ -299,7 +299,7 @@ mod tests { expires: Option, ) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index ca7f7ec2d..58ea8eb62 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -151,16 +151,48 @@ pub struct SummaryDescriptorRegistry { Option<( Arc, Arc, + std::collections::BTreeMap< + asap_types::sds::StoredOutputId, + asap_types::sds::StoredOutputReference, + >, )>, >, } impl SummaryDescriptorRegistry { + #[cfg(test)] pub fn install_catalog( &self, catalog: Arc, + ) -> Result<(), asap_types::summary_catalog::SummaryCatalogError> { + let outputs = catalog + .outputs + .keys() + .map(|id| (*id, catalog.output_reference(*id).unwrap())) + .collect(); + self.install_catalog_with_outputs(catalog, outputs) + } + + pub fn install_catalog_with_outputs( + &self, + catalog: Arc, + outputs: std::collections::BTreeMap< + asap_types::sds::StoredOutputId, + asap_types::sds::StoredOutputReference, + >, ) -> Result<(), asap_types::summary_catalog::SummaryCatalogError> { catalog.validate()?; + if outputs.len() != catalog.outputs.len() + || outputs + .iter() + .any(|(id, output)| catalog.output_reference(*id).ok().as_ref() != Some(output)) + { + return Err( + asap_types::summary_catalog::SummaryCatalogError::Descriptor( + "installed output bindings disagree with semantic definitions".into(), + ), + ); + } let reference = catalog.reference()?; let generation = Arc::new(asap_types::sds::CatalogGeneration { schema_version: reference.schema_version, @@ -168,7 +200,7 @@ impl SummaryDescriptorRegistry { plan_version: reference.plan_version, snapshot_sha256: reference.snapshot_sha256, }); - *self.authoritative_catalog.write().unwrap() = Some((catalog, generation)); + *self.authoritative_catalog.write().unwrap() = Some((catalog, generation, outputs)); Ok(()) } @@ -179,7 +211,7 @@ impl SummaryDescriptorRegistry { .read() .unwrap() .as_ref() - .map(|(catalog, _)| Arc::clone(catalog)) + .map(|(catalog, _, _)| Arc::clone(catalog)) } pub fn authoritative_snapshot( @@ -188,12 +220,39 @@ impl SummaryDescriptorRegistry { Arc, Arc, )> { - self.authoritative_catalog.read().unwrap().clone() + self.authoritative_catalog + .read() + .unwrap() + .as_ref() + .map(|(catalog, generation, _)| (Arc::clone(catalog), Arc::clone(generation))) + } + + pub fn stored_output_reference( + &self, + definition: asap_types::sds::StoredOutputId, + ) -> Option { + let installed = self.authoritative_catalog.read().unwrap(); + match installed.as_ref() { + Some((_, _, outputs)) => outputs.get(&definition).cloned(), + None => (!definition.fingerprint().is_unset()) + .then(|| asap_types::sds::StoredOutputReference::for_output(definition)), + } } pub fn bind(&self, metadata: SummarySeriesMetadata) -> Result { - let authoritative = self.authoritative_snapshot(); - let configured = if let Some((catalog, _)) = authoritative.as_ref() { + let authoritative = self.authoritative_catalog.read().unwrap().clone(); + let stored_output_reference = match &authoritative { + Some((_, _, outputs)) => Some( + outputs + .get(&metadata.policy_fp.into()) + .cloned() + .ok_or("definition has no selected stored output")?, + ), + None => (!metadata.policy_fp.is_unset()).then(|| { + asap_types::sds::StoredOutputReference::for_output(metadata.policy_fp.into()) + }), + }; + let configured = if let Some((catalog, _, _)) = authoritative.as_ref() { if metadata.policy_fp.is_unset() { return Err( "materialization identity is required by the installed SummaryCatalog".into(), @@ -252,20 +311,12 @@ impl SummaryDescriptorRegistry { } }; - let stored_output_reference = authoritative - .as_ref() - .map(|(catalog, _)| { - catalog - .output_reference(metadata.policy_fp.into()) - .map_err(|e| e.to_string()) - }) - .transpose()?; Ok(SdsBinding { stored_output_reference, metadata: Arc::new(metadata), summary_descriptor, data_descriptor, - catalog_generation: authoritative.map(|(_, generation)| generation), + catalog_generation: authoritative.map(|(_, generation, _)| generation), }) } @@ -356,7 +407,7 @@ mod tests { policy: u64, ) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.into(), group_by_keys: BTreeSet::from(["job".into()]), capability: None, diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index fc9062f1e..567852c80 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -1,86 +1,15 @@ -//! Hot-reloadable `StreamingConfig` state. +//! Generation-consistent physical plan installation and execution views. //! -//! Wraps a shared `StreamingConfig` in `arc_swap::ArcSwap` so an -//! external control plane can push a new config at runtime via -//! `POST /api/v1/streaming-config` without restarting the query -//! engine binary. -//! -//! ## How the pieces see the swap -//! -//! Runtime bootstrap readers share clones of the same `StreamingConfigHandle` -//! handle (internally `Arc>`), so they -//! observe the swap at the same instant: -//! -//! * **Writes** — atomic via `ArcSwap::store`. Lock-free; readers that -//! hold a stale snapshot finish their work with the old config and -//! drop it when the last reference goes out of scope. -//! * **IngestState** — re-snapshots per ingest batch -//! (`config_snapshot()`). New aggregations start receiving data on -//! the next batch. -//! * **Precompute workers** — read the handle directly in -//! `get_or_create_group_state()`. No message passing, no polling; -//! new agg_ids are visible the moment a worker tries to create a -//! `GroupState` for them. -//! -//! Query execution obtains its generation-consistent runtime configuration, -//! query plan, and catalog from `RuntimePhysicalPlan` instead of this handle. -//! -//! ## Config-upgrade contract for the control plane -//! -//! The recommended way for a control plane to upgrade a metric's sketch -//! parameters (or aggregation type) is **monotonic, non-reused -//! `aggregation_id`s plus time-based retention**: -//! -//! 1. Control plane decides to upgrade, e.g. `CMS(width=256)` → -//! `CMS(width=1024)` for `test_metric`. -//! 2. Control plane allocates a **new** `aggregation_id` (never reused), -//! e.g. the old id was 1, the new id is 17. -//! 3. Control plane POSTs a new `StreamingConfig` where the old id is -//! **removed** and the new id is **added**: -//! - before: `{1: CMS(width=256)}` -//! - after: `{17: CMS(width=1024)}` -//! 4. What happens on the backend, with zero additional code: -//! - `IngestState` stops routing data to agg_id 1 and starts -//! routing to agg_id 17 (metric-name match unchanged). -//! - Workers evict the now-orphaned `GroupState` entries for -//! agg_id 1 after their last windows drain -//! (`evict_orphaned_groups`). -//! - New `GroupState` entries for agg_id 17 are created on -//! demand, with the new `CMS(width=1024)` parameters. -//! - Store entries under agg_id 1 are **not deleted** on the -//! config swap; they persist until `persistence_delete_older_than_secs` -//! retention elapses, at which point the persistence layer's -//! time-based TTL sweep drops the corresponding parts. -//! 5. Query semantics during the transition: -//! - Before the swap: `ASAPQueryEngine` matches against agg_id 1. -//! - After the swap: `ASAPQueryEngine` matches against agg_id 17. -//! Historical data in the store under agg_id 1 is not joined -//! into the answer; the new sketch warms up from zero. -//! - Callers that need query continuity across parameter changes -//! should implement an overlap period at the control plane (keep -//! both ids in the config long enough for the new id to accrue -//! enough history) — this is a control-plane-side concern, not a -//! backend one. -//! -//! ## What the contract requires from the control plane -//! -//! * Assign `aggregation_id`s from a monotonically-increasing counter. -//! * Never reuse an `aggregation_id` after it has been removed from -//! a `StreamingConfig` push. -//! * Rely on the backend's `persistence_delete_older_than_secs` for -//! store cleanup — do not try to explicitly delete old agg_id data. -//! -//! Violating "never reuse" is safe in terms of correctness (the -//! backend creates a fresh `GroupState` either way), but it can -//! produce confusing store states where data under the same agg_id -//! spans multiple parameter generations. +//! Production consumers obtain the installed precompute program through the +//! active physical plan. Plan publication installs query and precompute bindings +//! together; a runtime lookup view cannot independently redefine computation. use std::collections::BTreeMap; use std::sync::Arc; use arc_swap::ArcSwap; -use crate::storage_engines::types::StreamingConfig; +use crate::storage_engines::types::InstalledPrecomputePlan; /// One immutable, generation-consistent runtime snapshot. Every execution /// subsystem must project its view from the same `Arc`. @@ -93,7 +22,7 @@ pub struct RuntimePhysicalPlan { pub summary_catalog: Option>, pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, pub transmission_plan: asap_types::producer_plan::TransmissionPlan, - pub streaming_config: Arc, + pub installed_precompute_plan: Arc, pub query_plan: Arc, pub storage_routing: Arc, } @@ -568,74 +497,69 @@ impl std::fmt::Debug for ActivePhysicalPlanHandle { } } -/// Streaming materialization view with two compatibility modes. Legacy mode -/// owns a swappable config; active-plan mode reads the config from the current -/// immutable runtime plan. In active-plan mode, `swap` only updates the legacy -/// backing slot and does not publish a new plan. Use plan activation to change -/// the authoritative configuration. +/// Precompute projection of the authoritative active physical plan. #[derive(Clone)] -pub struct StreamingConfigHandle { - inner: Arc>, - active: Option, +pub struct InstalledPrecomputePlanHandle { + source: InstalledPrecomputePlanSource, } -impl StreamingConfigHandle { - /// Construct with an initial `StreamingConfig`. Takes ownership — - /// callers who need to keep their own handle should `.clone()` the - /// `StreamingConfig` before calling `new`. - pub fn new(initial: StreamingConfig) -> Self { - Self { - inner: Arc::new(ArcSwap::new(Arc::new(initial))), - active: None, - } +#[derive(Clone)] +enum InstalledPrecomputePlanSource { + Active(ActivePhysicalPlanHandle), + #[cfg(test)] + Fixture(Arc>), +} + +impl InstalledPrecomputePlanHandle { + #[cfg(test)] + pub fn new(initial: InstalledPrecomputePlan) -> Self { + Self::from_arc(Arc::new(initial)) } - /// Construct from a pre-built `Arc` — useful - /// when the caller already has the config behind an `Arc` and - /// wants to avoid a redundant clone. - pub fn from_arc(initial: Arc) -> Self { + #[cfg(test)] + pub fn from_arc(initial: Arc) -> Self { Self { - inner: Arc::new(ArcSwap::new(initial)), - active: None, + source: InstalledPrecomputePlanSource::Fixture(Arc::new(ArcSwap::new(initial))), } } pub fn from_active_physical_plan(active: ActivePhysicalPlanHandle) -> Self { - let initial = active.active_snapshot().streaming_config.clone(); Self { - inner: Arc::new(ArcSwap::new(initial)), - active: Some(active), + source: InstalledPrecomputePlanSource::Active(active), } } - /// Return a cheap, cloneable snapshot of the current config. The - /// returned `Arc` is stable for the caller's lifetime — a - /// concurrent swap produces a new `Arc` and leaves this one alone. - pub fn snapshot(&self) -> Arc { - self.active - .as_ref() - .map(|a| a.active_snapshot().streaming_config.clone()) - .unwrap_or_else(|| self.inner.load_full()) + pub fn snapshot(&self) -> Arc { + match &self.source { + InstalledPrecomputePlanSource::Active(active) => { + active.active_snapshot().installed_precompute_plan.clone() + } + #[cfg(test)] + InstalledPrecomputePlanSource::Fixture(view) => view.load_full(), + } } pub fn active_physical_plan_snapshot(&self) -> Option> { - self.active.as_ref().map(|active| active.active_snapshot()) + match &self.source { + InstalledPrecomputePlanSource::Active(active) => Some(active.active_snapshot()), + #[cfg(test)] + InstalledPrecomputePlanSource::Fixture(_) => None, + } } - /// Atomically replace the current config. The previous `Arc` is - /// dropped when the last reader holding it goes out of scope. - /// Returns the `Arc` that was just replaced, for callers that - /// want to diff old vs new (e.g. to log agg_ids that were added - /// or removed). - pub fn swap(&self, new: StreamingConfig) -> Arc { - self.inner.swap(Arc::new(new)) + #[cfg(test)] + pub fn swap(&self, new: InstalledPrecomputePlan) -> Arc { + match &self.source { + InstalledPrecomputePlanSource::Fixture(view) => view.swap(Arc::new(new)), + InstalledPrecomputePlanSource::Active(_) => panic!("activate a complete physical plan"), + } } } -impl std::fmt::Debug for StreamingConfigHandle { +impl std::fmt::Debug for InstalledPrecomputePlanHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let snap = self.snapshot(); - f.debug_struct("StreamingConfigHandle") + f.debug_struct("InstalledPrecomputePlanHandle") .field( "num_agg_configs", &snap.materializations_by_policy_fingerprint.len(), @@ -650,10 +574,7 @@ pub use ActivePhysicalPlanHandle as HotReloadActivePhysicalPlan; #[deprecated(note = "Use RuntimePhysicalPlan")] pub use RuntimePhysicalPlan as ActivePhysicalPlan; -#[deprecated(note = "Use StreamingConfigHandle")] -pub use StreamingConfigHandle as HotReloadStreamingConfig; - -impl StreamingConfigHandle { +impl InstalledPrecomputePlanHandle { #[deprecated(note = "Use from_active_physical_plan")] pub fn from_active(active: ActivePhysicalPlanHandle) -> Self { Self::from_active_physical_plan(active) @@ -744,7 +665,7 @@ mod tests { }, rules: Vec::new(), }, - streaming_config: Arc::new(StreamingConfig::new(HashMap::new())), + installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(HashMap::new())), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id, plan_version, @@ -776,12 +697,12 @@ mod tests { ) } - /// Build a StreamingConfig from a list of marker `id`s. After PR 5 + /// Build a InstalledPrecomputePlan from a list of marker `id`s. After PR 5 /// the map key IS the policy fingerprint, derived from /// `metric_{id}`. We return both the config and the /// dummy-id→fingerprint mapping so the assertions below can look /// up entries. - fn cfg_with_ids(ids: &[u64]) -> (StreamingConfig, std::collections::HashMap) { + fn cfg_with_ids(ids: &[u64]) -> (InstalledPrecomputePlan, std::collections::HashMap) { let mut map = HashMap::new(); let mut id_to_fp = std::collections::HashMap::new(); for &id in ids { @@ -790,13 +711,13 @@ mod tests { id_to_fp.insert(id, fp); map.insert(fp, cfg); } - (StreamingConfig::new(map), id_to_fp) + (InstalledPrecomputePlan::new(map), id_to_fp) } #[test] fn snapshot_reflects_initial_config() { let (cfg, id_to_fp) = cfg_with_ids(&[1, 2, 3]); - let hr = StreamingConfigHandle::new(cfg); + let hr = InstalledPrecomputePlanHandle::new(cfg); let snap = hr.snapshot(); assert_eq!(snap.materializations_by_policy_fingerprint.len(), 3); assert!(snap @@ -808,7 +729,7 @@ mod tests { fn swap_replaces_config_atomically() { let (cfg1, id_to_fp1) = cfg_with_ids(&[1, 2]); let (cfg2, id_to_fp2) = cfg_with_ids(&[3, 4, 5]); - let hr = StreamingConfigHandle::new(cfg1); + let hr = InstalledPrecomputePlanHandle::new(cfg1); let old = hr.swap(cfg2); // Old snapshot still reflects pre-swap contents. assert_eq!(old.materializations_by_policy_fingerprint.len(), 2); @@ -830,7 +751,7 @@ mod tests { fn clones_share_underlying_swap() { let (cfg1, _) = cfg_with_ids(&[1]); let (cfg2, id_to_fp2) = cfg_with_ids(&[2, 3]); - let hr = StreamingConfigHandle::new(cfg1); + let hr = InstalledPrecomputePlanHandle::new(cfg1); let hr_clone = hr.clone(); hr.swap(cfg2); // The clone sees the swap because both handles share the @@ -845,7 +766,7 @@ mod tests { #[test] fn concurrent_readers_see_consistent_snapshot() { let (cfg1, _) = cfg_with_ids(&[1, 2]); - let hr = StreamingConfigHandle::new(cfg1); + let hr = InstalledPrecomputePlanHandle::new(cfg1); let hr_writer = hr.clone(); let writer = thread::spawn(move || { for i in 0..50 { diff --git a/data_plane/src/storage_engines/types/installed_precompute_plan.rs b/data_plane/src/storage_engines/types/installed_precompute_plan.rs new file mode 100644 index 000000000..4ca677707 --- /dev/null +++ b/data_plane/src/storage_engines/types/installed_precompute_plan.rs @@ -0,0 +1,174 @@ +//! Internal execution indexes derived exclusively from a validated DAG plan. +use anyhow::Result; +use std::collections::HashMap; +use std::ops::Index; + +use super::storage_backend::StorageBackend; +use asap_types::{PolicyRegistry, PrecomputeMaterialization}; + +#[derive(Debug, Clone)] +pub struct InstalledPrecomputePlan { + pub(crate) partitioning: crate::precompute_engine::partitioning::DagPartitioning, + pub(crate) raw_programs: + HashMap>, + pub(crate) precompute_plan: Option, + pub(crate) materializations_by_policy_fingerprint: HashMap, + pub(crate) storage_backend: StorageBackend, +} + +impl InstalledPrecomputePlan { + fn derived_view(materializations: HashMap) -> Self { + Self { + partitioning: Default::default(), + raw_programs: HashMap::new(), + precompute_plan: None, + materializations_by_policy_fingerprint: materializations, + storage_backend: StorageBackend::default(), + } + } + + /// Production construction always validates the executable DAG and bindings. + pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { + let materializations = plan.runtime_materializations()?; + let mut programs = HashMap::new(); + for config in materializations.values().filter(|config| { + config.derived_input.is_none() + && plan.ingest.protocol + == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 + }) { + let program = + crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) + .map_err(anyhow::Error::msg)?; + programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); + } + let mut view = Self::derived_view(materializations); + view.partitioning = + crate::precompute_engine::partitioning::DagPartitioning::from_plan(&plan); + view.precompute_plan = Some(plan); + view.raw_programs = programs; + Ok(view) + } + + // Isolated kernel/storage fixtures can omit a physical installation. This + // constructor is absent from the production library and binary. + #[cfg(test)] + pub fn new(materializations: HashMap) -> Self { + Self::derived_view(materializations) + } + + #[cfg(test)] + pub fn with_storage_backend( + materializations: HashMap, + storage_backend: StorageBackend, + ) -> Self { + let mut view = Self::derived_view(materializations); + view.storage_backend = storage_backend; + view + } + + pub fn stored_output_reference( + &self, + definition: asap_types::sds::StoredOutputId, + ) -> Option { + let selected = self.precompute_plan.as_ref().and_then(|plan| { + plan.schemas + .iter() + .find(|schema| schema.materialization == definition) + .map(|schema| schema.stored_output_reference.clone()) + }); + #[cfg(test)] + let selected = selected.or_else(|| { + let materializations = self + .materializations_by_policy_fingerprint + .values() + .cloned() + .collect::>(); + asap_types::summary_catalog::SummaryCatalog::from_materializations( + 0, + 0, + &materializations, + ) + .ok()? + .output_reference(definition) + .ok() + }); + selected + } + + pub fn plan(&self) -> &asap_types::precompute_plan::PrecomputePlan { + self.precompute_plan + .as_ref() + .expect("installed plan has an authoritative source") + } + + pub fn storage_backend(&self) -> StorageBackend { + self.storage_backend + } + + pub fn get_aggregation_config(&self, fingerprint: u64) -> Option<&PrecomputeMaterialization> { + self.materializations_by_policy_fingerprint + .get(&fingerprint) + } + + pub fn materializations(&self) -> &HashMap { + &self.materializations_by_policy_fingerprint + } + + pub fn contains(&self, fingerprint: u64) -> bool { + self.materializations_by_policy_fingerprint + .contains_key(&fingerprint) + } + + pub fn policy_registry(&self) -> PolicyRegistry { + PolicyRegistry::from_configs( + self.materializations_by_policy_fingerprint + .values() + .cloned(), + ) + } +} + +impl Index for InstalledPrecomputePlan { + type Output = PrecomputeMaterialization; + fn index(&self, fingerprint: u64) -> &Self::Output { + &self.materializations_by_policy_fingerprint[&fingerprint] + } +} + +impl Default for InstalledPrecomputePlan { + fn default() -> Self { + use control_plane::physical::compiler::{ + PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION, + }; + let envelope = PlanEnvelope { + plan_id: 0, + plan_version: 0, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "empty-installation".into(), + }; + Self::from_precompute_plan( + PrecomputePlan::build(envelope, vec![], &[]).expect("valid empty plan"), + ) + .expect("valid empty installation") + } +} + +#[cfg(test)] +mod tests { + // Flat lists cannot enter through the authoritative physical-plan document. + #[test] + fn rejects_flat_aggregation_documents() { + for text in [r#"{"aggregation_configs":{}}"#, "aggregations: []"] { + assert!( + serde_yaml::from_str::( + text + ) + .is_err() + ); + } + } +} diff --git a/data_plane/src/storage_engines/types/mod.rs b/data_plane/src/storage_engines/types/mod.rs index 562d08d7f..9956f5b46 100644 --- a/data_plane/src/storage_engines/types/mod.rs +++ b/data_plane/src/storage_engines/types/mod.rs @@ -8,18 +8,18 @@ pub mod enums; pub mod hot_reload_config; +pub mod installed_precompute_plan; pub mod precomputed_output; pub mod storage_backend; -pub mod streaming_config; pub use asap_physical_operators::key_by_label_values::*; pub use asap_physical_operators::measurement::*; pub use asap_physical_operators::traits::*; pub use enums::*; pub use hot_reload_config::*; +pub use installed_precompute_plan::*; pub use precomputed_output::*; pub use storage_backend::*; -pub use streaming_config::*; // Cross-module re-export of asap_types data types so callers can // write `crate::storage_engines::types::PrecomputeMaterialization` instead of diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 28e71b4c3..8644f37da 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -44,7 +44,9 @@ pub struct PrecomputedOutput { /// Physical lifetime chosen before execution; never re-resolve a queued /// fragment against a newer logical-series mapping. #[serde(skip)] - pub series_id: Option, + pub storage_handle: Option, + #[serde(skip)] + pub stored_output_reference: Option, #[serde(skip)] pub catalog_generation: Option>, #[serde(skip)] @@ -86,7 +88,8 @@ impl PrecomputedOutput { policy_fp: PolicyFingerprint, ) -> Self { Self { - series_id: None, + storage_handle: None, + stored_output_reference: None, catalog_generation: None, input_revision: None, start_timestamp, @@ -115,7 +118,8 @@ impl PrecomputedOutput { policy_fp: PolicyFingerprint, ) -> Self { Self { - series_id: None, + storage_handle: None, + stored_output_reference: None, catalog_generation: None, input_revision: None, start_timestamp, diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs deleted file mode 100644 index 95055c2ea..000000000 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ /dev/null @@ -1,211 +0,0 @@ -use anyhow::Result; -use serde::{Deserialize, Serialize}; -use serde_yaml::Value; -use std::collections::HashMap; -use std::fs::File; -use std::io::BufReader; -use std::ops::Index; - -use asap_types::{MonitorSpec, PolicyRegistry, PrecomputeMaterialization}; - -use super::storage_backend::StorageBackend; - -/// DAG installation plus a derived in-memory routing index. The flat index is -/// never serialized as executable configuration. Raw programs are validated -/// and shared once per installed producer across all of its population states. -#[derive(Debug, Clone, Serialize)] -pub struct StreamingConfig { - #[serde(skip)] - pub(crate) raw_programs: - HashMap>, - /// Authoritative execution configuration: Planner DAGs and physical bindings. - pub precompute_plan: Option, - #[serde(skip)] - pub materializations_by_policy_fingerprint: HashMap, - /// Phase-5 capability-routing axis: which storage tier serves this - /// per-metric runtime config. The controller pushes this when planning - /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 - /// configs decode with `#[serde(default)]` to `SketchStore` so - /// existing deploys keep dispatching to `ASAPQueryEngine`. - #[serde(default)] - pub storage_backend: StorageBackend, - /// Continuous-distributed-monitoring threshold specs the data-plane - /// coordinator should serve. Defaults to empty so existing configs (and the - /// vast majority of deploys, which run no monitors) decode unchanged. - #[serde(default)] - pub monitors: Vec, -} - -// Flat aggregation lists are deliberately not an accepted execution document. -impl<'de> Deserialize<'de> for StreamingConfig { - fn deserialize>(deserializer: D) -> Result { - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct Document { - precompute_plan: asap_types::precompute_plan::PrecomputePlan, - #[serde(default)] - storage_backend: StorageBackend, - #[serde(default)] - monitors: Vec, - } - let doc = Document::deserialize(deserializer)?; - let mut config = - Self::from_precompute_plan(doc.precompute_plan).map_err(serde::de::Error::custom)?; - config.storage_backend = doc.storage_backend; - config.monitors = doc.monitors; - Ok(config) - } -} - -impl StreamingConfig { - pub fn new( - materializations_by_policy_fingerprint: HashMap, - ) -> Self { - Self { - raw_programs: HashMap::new(), - precompute_plan: None, - materializations_by_policy_fingerprint, - storage_backend: StorageBackend::default(), - monitors: Vec::new(), - } - } - - /// Build the routing projection only after validating the DAG installation. - pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { - let materializations = plan.runtime_materializations()?; - let mut programs = HashMap::new(); - for config in materializations.values().filter(|c| { - c.derived_input.is_none() - && plan.ingest.protocol - == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 - }) { - let program = - crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) - .map_err(anyhow::Error::msg)?; - programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); - } - let mut view = Self::new(materializations); - view.precompute_plan = Some(plan); - view.raw_programs = programs; - Ok(view) - } - - /// CDM monitor specs the data-plane coordinator should serve (may be empty). - pub fn monitors(&self) -> &[MonitorSpec] { - &self.monitors - } - - /// Phase-5 constructor: build with an explicit storage-backend pin. - /// Used by the controller-driven plan-push path; tests typically - /// stay on `Self::new(...)` and let the default land. - pub fn with_storage_backend( - materializations_by_policy_fingerprint: HashMap, - storage_backend: StorageBackend, - ) -> Self { - Self { - raw_programs: HashMap::new(), - precompute_plan: None, - materializations_by_policy_fingerprint, - storage_backend, - monitors: Vec::new(), - } - } - - /// Read-only access to the storage backend pinned at construction - /// time. The Phase-5 router consults this to pick which engine - /// answers a query. - pub fn storage_backend(&self) -> StorageBackend { - self.storage_backend - } - - pub fn get_aggregation_config( - &self, - aggregation_id: u64, - ) -> Option<&PrecomputeMaterialization> { - self.materializations_by_policy_fingerprint - .get(&aggregation_id) - } - - pub fn materializations(&self) -> &HashMap { - &self.materializations_by_policy_fingerprint - } - - pub fn contains(&self, aggregation_id: u64) -> bool { - self.materializations_by_policy_fingerprint - .contains_key(&aggregation_id) - } - - /// Derived content-addressed view. Builds a [`PolicyRegistry`] keyed - /// on [`asap_types::PolicyFingerprint`] — the merged-sid-identity-chain - /// replacement for the `aggregation_id`-keyed lookup. Cheap (O(N) - /// over `materializations_by_policy_fingerprint.len()`); call at swap time, not per - /// query, if it shows up in hot-path profiles. - /// - /// Dual-keyed transition: this method exists alongside the legacy - /// `get_aggregation_config(aggregation_id)` so callers can migrate - /// one at a time. The two views are derived from the same source — - /// they can never disagree. - pub fn policy_registry(&self) -> PolicyRegistry { - PolicyRegistry::from_configs( - self.materializations_by_policy_fingerprint - .values() - .cloned(), - ) - } - - pub fn from_yaml_file(yaml_file: &str) -> Result { - let file = File::open(yaml_file)?; - let reader = BufReader::new(file); - let data: Value = serde_yaml::from_reader(reader)?; - - Self::from_yaml_data(&data) - } - - /// Build from the streaming-config YAML alone. Per-aggregation - /// `numAggregatesToRetain` is read directly from each - /// aggregation's YAML entry; the old InferenceConfig indirection - /// (operator-authored query→agg_ids YAML feeding a retention_map) - /// is gone — the controller drives capability matching dynamically. - pub fn from_yaml_data(data: &Value) -> Result { - serde_yaml::from_value(data.clone()).map_err(Into::into) - } -} - -impl Index for StreamingConfig { - type Output = PrecomputeMaterialization; - - fn index(&self, aggregation_id: u64) -> &Self::Output { - &self.materializations_by_policy_fingerprint[&aggregation_id] - } -} - -impl Default for StreamingConfig { - fn default() -> Self { - Self::new(HashMap::new()) - } -} - -impl StreamingConfig { - #[deprecated(note = "Use materializations")] - pub fn get_all_aggregation_configs(&self) -> &HashMap { - self.materializations() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // Old flat lists cannot become execution authority through JSON or YAML. - #[test] - fn rejects_flat_aggregation_documents() { - for text in [ - r#"{"aggregation_configs":{}}"#, - "aggregations: []", - "aggregations: [{aggregationType: Sum, metric: m}]", - ] { - let yaml = serde_yaml::from_str(text).unwrap(); - assert!(StreamingConfig::from_yaml_data(&yaml).is_err()); - } - } -} diff --git a/data_plane/src/tests/prometheus_forwarding_tests.rs b/data_plane/src/tests/prometheus_forwarding_tests.rs index ff205f4e0..bfbe765ff 100644 --- a/data_plane/src/tests/prometheus_forwarding_tests.rs +++ b/data_plane/src/tests/prometheus_forwarding_tests.rs @@ -2,7 +2,7 @@ use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::{ASAPQueryEngine, QueryForwardingPolicy}; #[cfg(test)] -use crate::storage_engines::types::{QueryLanguage, StreamingConfig}; +use crate::storage_engines::types::{InstalledPrecomputePlan, QueryLanguage}; use reqwest::Client; use serde_json::Value; use std::sync::{ @@ -110,7 +110,7 @@ async fn setup_test_server(prometheus_port: u16) -> (HttpServer, u16) { ), }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let idx = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -190,7 +190,7 @@ async fn test_forwarding_disabled() { ), }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let idx = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -278,7 +278,7 @@ async fn test_prometheus_server_unreachable() { ), }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let idx = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 974673941..3b0a540fb 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -9,8 +9,8 @@ use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; use crate::query_engines::query_result::InstantVectorElement; use crate::storage_engines::types::{ - AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, QueryLanguage, - StreamingConfig, WindowKind, + AggregationType, InstalledPrecomputePlan, KeyByLabelValues, PrecomputeMaterialization, + PrecomputedOutput, QueryLanguage, WindowKind, }; use crate::AggregateCore; use asap_types::KeyByLabelNames; @@ -120,12 +120,12 @@ pub fn create_engine_single_pop_with_aggregated( let agg_id = agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(agg_id, agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = @@ -133,10 +133,10 @@ pub fn create_engine_single_pop_with_aggregated( let resolver = std::sync::Arc::new(SeriesIdResolver::new()); // Insert data into SketchStore via the canonical helper (M2.3.6e). - let agg_cfg = streaming_config + let agg_cfg = installed_precompute_plan .get_aggregation_config(agg_id) .cloned() - .expect("agg config must be in streaming_config"); + .expect("agg config must be in installed_precompute_plan"); let timestamp = 1_000_000_u64; for (label_values_opt, acc) in data { let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); @@ -242,23 +242,23 @@ pub fn create_engine_dual_input( let keys_id = keys_agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg_1 = streaming_config + let agg_cfg_1 = installed_precompute_plan .get_aggregation_config(value_id) .cloned() .expect("value agg config"); - let agg_cfg_2 = streaming_config + let agg_cfg_2 = installed_precompute_plan .get_aggregation_config(keys_id) .cloned() .expect("keys agg config"); @@ -372,22 +372,22 @@ pub fn create_engine_two_metrics( let id_b = agg_config_b.policy_fp_u64(); materializations_by_policy_fingerprint.insert(id_b, agg_config_b); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg_1 = streaming_config + let agg_cfg_1 = installed_precompute_plan .get_aggregation_config(id_a) .cloned() .expect("agg a"); - let agg_cfg_2 = streaming_config + let agg_cfg_2 = installed_precompute_plan .get_aggregation_config(id_b) .cloned() .expect("agg b"); @@ -483,12 +483,12 @@ pub fn create_engine_three_metrics( materializations_by_policy_fingerprint.insert(id, cfg); } - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = @@ -497,7 +497,7 @@ pub fn create_engine_three_metrics( let agg_cfgs: Vec<_> = ids .iter() .map(|id| { - streaming_config + installed_precompute_plan .get_aggregation_config(*id) .cloned() .expect("agg present") @@ -567,18 +567,18 @@ pub fn create_engine_multi_timestamp( let agg_id = agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(agg_id, agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = streaming_config + let agg_cfg = installed_precompute_plan .get_aggregation_config(agg_id) .cloned() .expect("agg"); @@ -645,18 +645,18 @@ pub fn create_engine_multi_timestamp_with_window( let agg_id = agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(agg_id, agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = streaming_config + let agg_cfg = installed_precompute_plan .get_aggregation_config(agg_id) .cloned() .expect("agg"); diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs deleted file mode 100644 index 6f33ca87b..000000000 --- a/data_plane/src/utils/file_io.rs +++ /dev/null @@ -1,52 +0,0 @@ -use crate::storage_engines::types::StreamingConfig; -use anyhow::{Context, Result}; - -pub fn read_streaming_config(yaml_file: &str) -> Result { - let yaml_data = std::fs::read_to_string(yaml_file) - .with_context(|| format!("Failed to read YAML file: {yaml_file}"))?; - let yaml_data: serde_yaml::Value = serde_yaml::from_str(&yaml_data) - .with_context(|| format!("Failed to parse YAML file: {yaml_file}"))?; - - let config = StreamingConfig::from_yaml_data(&yaml_data) - .with_context(|| format!("Failed to parse YAML config from: {yaml_file}"))?; - - Ok(config) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use tempfile::NamedTempFile; - - #[test] - fn flat_streaming_file_is_rejected() { - // PR 5: `aggregationId: 1` is silently dropped on read — the - // streaming-config map key is the policy fingerprint derived - // from content. The legacy field stays in this fixture to - // exercise the "ignored cleanly" path. - let streaming_yaml_content = r#" -aggregations: -- aggregationId: 1 - aggregationSubType: '' - aggregationType: DatasketchesKLL - labels: - aggregated: [] - grouping: - - instance - - job - rollup: [] - metric: fake_metric_total - parameters: - K: 200 - spatialFilter: '' - windowSize: 10 - numAggregatesToRetain: 6 -"#; - - let mut streaming_temp_file = NamedTempFile::new().unwrap(); - write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); - - assert!(read_streaming_config(streaming_temp_file.path().to_str().unwrap()).is_err()); - } -} diff --git a/data_plane/src/utils/mod.rs b/data_plane/src/utils/mod.rs index 5d620636b..18e84ff2b 100644 --- a/data_plane/src/utils/mod.rs +++ b/data_plane/src/utils/mod.rs @@ -1,5 +1,3 @@ -pub mod file_io; pub mod http; -pub use file_io::*; pub use http::*; diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index 9cee54846..4340e6154 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -6,7 +6,6 @@ //! answers are independently computed from those raw fixtures. use std::collections::{HashMap, HashSet}; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -60,7 +59,6 @@ struct Backend { client: reqwest::Client, query_base: String, otlp_url: String, - _config: tempfile::NamedTempFile, _output_dir: tempfile::TempDir, } @@ -114,18 +112,9 @@ async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create data-plane output directory"); - let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); + let mut physical = tempfile::NamedTempFile::new().unwrap(); let mut install = physical_fixture::artifact_from_materializations(vec![materialization.clone()]); - serde_yaml::to_writer( - &mut config, - &serde_json::json!({ - "precompute_plan": install.precompute_plan, - }), - ) - .unwrap(); - config.flush().unwrap(); - let mut physical = tempfile::NamedTempFile::new().unwrap(); for rule in &mut install.transmission_plan.rules { if matches!( install @@ -148,8 +137,6 @@ async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) let mut command = Command::new(env!("CARGO_BIN_EXE_data_plane")); command - .arg("--streaming-config") - .arg(config.path()) .arg("--physical-plan") .arg(physical.path()) .arg("--http-port") @@ -189,7 +176,6 @@ async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) client, query_base, otlp_url: format!("http://127.0.0.1:{otlp_http_port}/v1/metrics"), - _config: config, _output_dir: output_dir, }; } @@ -296,23 +282,10 @@ fn scalar_values(response: &Value) -> Vec<(HashMap, f64)> { } fn config(metric: &str, kind: &str, parameters: &str) -> asap_types::PrecomputeMaterialization { - use asap_types::{KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - PrecomputeMaterialization::new( + physical_fixture::materialization( + metric, kind.parse().unwrap(), - String::new(), serde_yaml::from_str(parameters).unwrap(), - KeyByLabelNames::new(vec!["service".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 1, - 1, - WindowKind::Tumbling, - String::new(), - metric.into(), - None, - None, - None, ) } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 99fae4b0a..da96d9365 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -27,6 +27,8 @@ mod distinct_planning_process; mod durable_summary_process; #[path = "support/immutable_maintenance_process.rs"] mod immutable_maintenance_process; +#[path = "support/revisable_maintenance_process.rs"] +mod revisable_maintenance_process; #[path = "support/current_series_process.rs"] mod current_series_process; @@ -399,18 +401,10 @@ async fn certified_kll_state_to_query_oracle() { let port = unused_port(); let otlp_port = unused_port(); let grpc_port = unused_port(); - let mut bootstrap_config = tempfile::NamedTempFile::new().unwrap(); - serde_json::to_writer( - &mut bootstrap_config, - &serde_json::json!({"precompute_plan": install.precompute_plan}), - ) - .unwrap(); let mut child = ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) .args(["--physical-plan"]) .arg(artifact_file.path()) - .arg("--streaming-config") - .arg(bootstrap_config.path()) .args(["--http-port", &port.to_string(), "--output-dir"]) .arg(output.path()) .args([ diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index 910025999..3832f0b0a 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -5,8 +5,8 @@ //! data plane, sends a modified-OTLP DDSketch, and verifies the resulting //! PromQL value. No server or planner is constructed in the test process. -#[path = "support/streaming_config.rs"] -mod streaming_config; +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; @@ -376,10 +376,10 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { let output_dir = tempfile::tempdir().expect("create data-plane output directory"); let mut bootstrap = tempfile::NamedTempFile::new().expect("create bootstrap config"); - serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let data_child = Command::new(env!("CARGO_BIN_EXE_data_plane")) - .arg("--streaming-config") + .arg("--physical-plan") .arg(bootstrap.path()) .arg("--http-port") .arg(port(&data_api).to_string()) @@ -418,7 +418,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .env("CONTROLLER_GRPC_ADDR", &control_grpc) .env( "CONTROLLER_BACKEND_ENDPOINT", - format!("{data_base}/api/v1/streaming-config"), + format!("{data_base}/api/v1/physical-plan"), ) .env( "CONTROLLER_WORKLOADS", @@ -539,7 +539,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { assert_eq!(publication["collector_ids"][0], "whole-e2e-collector"); let active: serde_json::Value = client - .get(format!("{data_base}/api/v1/streaming-config")) + .get(format!("{data_base}/api/v1/physical-plan/status")) .send() .await .expect("read installed streaming config") @@ -547,10 +547,11 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .expect("decode installed streaming config"); assert_eq!( - active["aggregation_count"], 1, + active["materializations"].as_array().unwrap().len(), + 1, "physical plan was not installed: {active}" ); - let installed_aggregation = active["streaming_config"]["precompute_plan"]["materializations"] + let installed_aggregation = active["precompute_plan"]["materializations"] .as_array() .and_then(|configs| configs.first()) .expect("installed aggregation details"); diff --git a/data_plane/tests/clickhouse_differential_e2e.rs b/data_plane/tests/clickhouse_differential_e2e.rs index 471b71530..b9a6523d4 100644 --- a/data_plane/tests/clickhouse_differential_e2e.rs +++ b/data_plane/tests/clickhouse_differential_e2e.rs @@ -2,8 +2,8 @@ //! //! Set `CLICKHOUSE_URL` (for example `http://127.0.0.1:8123`) to run it. -#[path = "support/streaming_config.rs"] -mod streaming_config; +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; use std::{ collections::HashMap, @@ -65,7 +65,7 @@ async fn spawn_backend( let client = reqwest::Client::new(); let mut command = Command::new(env!("CARGO_BIN_EXE_data_plane")); command - .arg("--streaming-config") + .arg("--physical-plan") .arg(bootstrap) .arg("--http-port") .arg(api_port.to_string()) @@ -347,7 +347,7 @@ async fn run_mixed_aggregate(aggregate: &str) { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -542,7 +542,7 @@ async fn moving_windows_match_clickhouse_after_automatic_publication() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -726,7 +726,7 @@ async fn collection_sql_executes_local_elements_after_typed_exact_leaf() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - serde_yaml::to_writer(&mut bootstrap, &streaming_config::empty()).unwrap(); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), diff --git a/data_plane/tests/clickhouse_q05_process_e2e.rs b/data_plane/tests/clickhouse_q05_process_e2e.rs index f358dea8c..57d0fa9dd 100644 --- a/data_plane/tests/clickhouse_q05_process_e2e.rs +++ b/data_plane/tests/clickhouse_q05_process_e2e.rs @@ -1,6 +1,9 @@ //! Optional OS-process E2E for a q05-class bounded SQL max query. //! Run with `CLICKHOUSE_URL=http://127.0.0.1:8123 cargo test -p data_plane --test clickhouse_q05_process_e2e`. +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; + use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; use planner_types::pre_asap::{Column, DataType, Schema}; use std::io::Read; @@ -196,18 +199,17 @@ async fn q05_sql_is_planned_backfilled_and_served_warm_by_backend_process() { while sql_port == http_port { sql_port = free_port(); } - let streaming_config = format!( - "{}/examples/promql/streaming_config.yaml", - env!("CARGO_MANIFEST_DIR") - ); + let mut physical = tempfile::NamedTempFile::new().unwrap(); + serde_json::to_writer(&mut physical, &empty_physical_plan::empty()).unwrap(); + let physical_path = physical.path().to_str().unwrap(); let output_dir = tempfile::tempdir().unwrap(); let output_dir_arg = output_dir.path().to_str().unwrap().to_owned(); let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_data_plane")); command.args([ "--http-port", &http_port.to_string(), - "--streaming-config", - &streaming_config, + "--physical-plan", + physical_path, "--clickhouse-http-port", &sql_port.to_string(), "--clickhouse-url", diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index 945a82aec..8a6d59269 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -124,38 +124,18 @@ async fn production_binary_ingests_ddsketch_and_answers_promql() { let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); - let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - let materialization = asap_types::PrecomputeMaterialization::new( - asap_types::AggregationType::DDSketch, - String::new(), - std::collections::HashMap::from([("relative_accuracy".into(), serde_json::json!(0.01))]), - asap_types::KeyByLabelNames::new(vec!["service".into()]), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), - 1, - 1, - asap_types::WindowKind::Tumbling, - String::new(), - "component_process_e2e_latency_ms".into(), - None, - None, - None, - ); - let install = physical_fixture::artifact_from_materializations(vec![materialization]); - serde_yaml::to_writer( - &mut config, - &serde_json::json!({"precompute_plan": install.precompute_plan}), - ) - .unwrap(); + let install = + physical_fixture::artifact_from_materializations(vec![physical_fixture::materialization( + "component_process_e2e_latency_ms", + asap_types::AggregationType::DDSketch, + [("relative_accuracy".into(), serde_json::json!(0.01))].into(), + )]); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(physical.path()) - .arg("--streaming-config") - .arg(config.path()) .arg("--http-port") .arg(query_port.to_string()) .arg("--output-dir") diff --git a/data_plane/tests/disable_query_forwarding_process_e2e.rs b/data_plane/tests/disable_query_forwarding_process_e2e.rs index 529134fc9..2cc1e9cf7 100644 --- a/data_plane/tests/disable_query_forwarding_process_e2e.rs +++ b/data_plane/tests/disable_query_forwarding_process_e2e.rs @@ -1,7 +1,7 @@ //! The production CLI's no-forwarding mode keeps query traffic inside the backend. -#[path = "support/streaming_config.rs"] -mod streaming_config; +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; @@ -63,12 +63,12 @@ async fn cli_mode_blocks_instant_and_range_forwarding() { }); let mut config = tempfile::NamedTempFile::new().unwrap(); - serde_yaml::to_writer(&mut config, &streaming_config::empty()).unwrap(); + serde_json::to_writer(&mut config, &empty_physical_plan::empty()).unwrap(); let output = tempfile::tempdir().unwrap(); let port = unused_port(); let mut child = ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) - .arg("--streaming-config") + .arg("--physical-plan") .arg(config.path()) .args([ "--disable-query-forwarding", diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 21dc9d6cf..ecc9d51aa 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -145,7 +145,7 @@ async fn post_full_config( } use control_plane::types::WorkloadCharacteristics; -use data_plane::storage_engines::types::HotReloadStreamingConfig; +use data_plane::storage_engines::types::InstalledPrecomputePlanHandle; use serde_json::Value as JsonValue; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; @@ -246,7 +246,7 @@ fn _wc_anchor() -> WorkloadCharacteristics { /// Full test stack: PrecomputeEngine + SketchStoreSink + OtlpReceiver + /// HttpServer, all sharing the same `SketchStore` and -/// `HotReloadStreamingConfig` so a controller-posted streaming-config +/// `InstalledPrecomputePlanHandle` so a controller-posted streaming-config /// is visible to the engine's accumulator routing, the engine's window /// outputs land in `SketchStore`, and the query engine reads from the /// same store. @@ -278,7 +278,7 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack let active = data_plane::storage_engines::types::HotReloadActivePhysicalPlan::new( physical_fixture::bootstrap(), ); - let hot_reload = HotReloadStreamingConfig::from_active_physical_plan(active.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_active_physical_plan(active.clone()); let series_resolver = Arc::new(SeriesIdResolver::new()); // SketchStoreSink writes precompute output back into SketchStore so @@ -736,7 +736,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // * Modified-OTLP `DdSketchDataPoint` wire encoding + the backend's // OTLP HTTP receiver accept the payload (no 4xx/5xx). // * The full stack (PrecomputeEngine + SketchStoreSink + OtlpReceiver -// + HttpServer all sharing SketchStore + HotReloadStreamingConfig) +// + HttpServer all sharing SketchStore + InstalledPrecomputePlanHandle) // comes up and stays up under POST + query traffic. // * The OTLP-ingested sketch lands in `SketchStore` keyed by the // right `PolicyFingerprint` (or via the `instances_matching` diff --git a/data_plane/tests/monitor_process_e2e.rs b/data_plane/tests/monitor_process_e2e.rs index 2a515ad0b..920d44281 100644 --- a/data_plane/tests/monitor_process_e2e.rs +++ b/data_plane/tests/monitor_process_e2e.rs @@ -4,8 +4,8 @@ //! data-plane executable, register, report different rates, and receive //! differentiated sampling grants over bidirectional gRPC streams. -#[path = "support/streaming_config.rs"] -mod streaming_config; +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; @@ -112,14 +112,18 @@ async fn production_coordinator_differentiates_edge_sampling_grants() { let monitor_port = unused_port(); let output_dir = tempfile::tempdir().expect("create output directory"); let mut config = tempfile::NamedTempFile::new().expect("create monitor config"); - let mut startup = streaming_config::empty(); - startup["monitors"] = serde_json::json!([{ - "agg_id": 1, "key": "", "tau": 5000.0, "epsilon": 0.05, "window_ms": 60000 - }]); - serde_yaml::to_writer(&mut config, &startup).unwrap(); + std::io::Write::write_all( + &mut config, + b"- agg_id: 1\n key: ''\n tau: 5000.0\n epsilon: 0.05\n window_ms: 60000\n", + ) + .unwrap(); + let mut physical = tempfile::NamedTempFile::new().unwrap(); + serde_json::to_writer(&mut physical, &empty_physical_plan::empty()).unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) - .arg("--streaming-config") + .arg("--physical-plan") + .arg(physical.path()) + .arg("--monitor-specs") .arg(config.path()) .arg("--http-port") .arg(query_port.to_string()) diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index fdda417eb..a3c6359c5 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -191,13 +191,9 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); - let bootstrap = output_dir.path().join("bootstrap.json"); - std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(physical.path()) - .arg("--streaming-config") - .arg(&bootstrap) .arg("--http-port") .arg(query_port.to_string()) .arg("--output-dir") @@ -393,14 +389,10 @@ async fn bound_sds_keeps_hot_and_rebuild_outputs_and_groups_isolated() { let directory = tempfile::tempdir().unwrap(); let plan = directory.path().join("plan.json"); std::fs::write(&plan, serde_json::to_vec(&install).unwrap()).unwrap(); - let bootstrap = directory.path().join("bootstrap.json"); - std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); let mut child = ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(&plan) - .arg("--streaming-config") - .arg(&bootstrap) .arg("--http-port") .arg(query_port.to_string()) .arg("--output-dir") diff --git a/data_plane/tests/support/durable_summary_process.rs b/data_plane/tests/support/durable_summary_process.rs index 909752886..604157b99 100644 --- a/data_plane/tests/support/durable_summary_process.rs +++ b/data_plane/tests/support/durable_summary_process.rs @@ -32,18 +32,14 @@ async fn persisted_summary_restarts_without_live_reregistration() { }; let directory = tempfile::tempdir().unwrap(); let artifact = directory.path().join("plan.json"); - let bootstrap = directory.path().join("bootstrap.json"); let disk = directory.path().join("disk"); std::fs::create_dir_all(&disk).unwrap(); std::fs::write(&artifact, serde_json::to_vec(&install).unwrap()).unwrap(); - std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); let spawn = |port: u16| { ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(&artifact) - .arg("--streaming-config") - .arg(&bootstrap) .arg("--http-port") .arg(port.to_string()) .arg("--output-dir") diff --git a/data_plane/tests/support/empty_physical_plan.rs b/data_plane/tests/support/empty_physical_plan.rs new file mode 100644 index 000000000..0e453d85b --- /dev/null +++ b/data_plane/tests/support/empty_physical_plan.rs @@ -0,0 +1,31 @@ +//! Empty authoritative installation for tests that publish a workload later. +pub fn empty() -> asap_types::plan_publication::PhysicalPlanInstallRequest { + use control_plane::physical::compiler::*; + let envelope = PlanEnvelope { + plan_id: 0, + plan_version: 0, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "empty-bootstrap".into(), + }; + let catalog = + control_plane::physical::summary_catalog::SummaryCatalog::from_materializations(0, 0, &[]) + .unwrap(); + let mut precompute_plan = PrecomputePlan::build(envelope.clone(), vec![], &[]).unwrap(); + precompute_plan.summary_catalog = Some(catalog.reference().unwrap()); + let mut transmission_plan = + build_transmission_plan(envelope, &precompute_plan, &Default::default()).unwrap(); + transmission_plan.summary_catalog = precompute_plan.summary_catalog.clone(); + asap_types::plan_publication::PhysicalPlanInstallRequest { + summary_catalog: catalog, + precompute_plan, + transmission_plan, + query_plan: asap_types::query_plan::QueryPlan::empty(), + collector_plans: vec![], + storage_routing: None, + adaptation_evidence: vec![], + } +} diff --git a/data_plane/tests/support/immutable_maintenance_process.rs b/data_plane/tests/support/immutable_maintenance_process.rs index eeeee37db..1aa1fefc5 100644 --- a/data_plane/tests/support/immutable_maintenance_process.rs +++ b/data_plane/tests/support/immutable_maintenance_process.rs @@ -103,18 +103,14 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { eprintln!("IMMUTABLE_PROCESS_ARTIFACT {}", directory.path().display()); directory.disable_cleanup(true); let artifact = directory.path().join("plan.json"); - let bootstrap = directory.path().join("bootstrap.json"); let disk = directory.path().join("disk"); std::fs::create_dir_all(&disk).unwrap(); std::fs::write(&artifact, serde_json::to_vec(&install).unwrap()).unwrap(); - std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); let spawn = |port: u16| { ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(&artifact) - .arg("--streaming-config") - .arg(&bootstrap) .args(["--http-port", &port.to_string(), "--output-dir"]) .arg(directory.path()) .arg("--enable-remote-write") diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index 609176084..1b9b03823 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -4,22 +4,42 @@ use control_plane::{physical::compiler::*, query_plan::*}; use data_plane::{ drivers::query::servers::http::PhysicalPlanInstallRequest, - storage_engines::types::{ActivePhysicalPlan, BackendStorageRouting, StreamingConfig}, + storage_engines::types::{ActivePhysicalPlan, BackendStorageRouting, InstalledPrecomputePlan}, }; use std::{collections::BTreeMap, sync::Arc}; -pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { - artifact_from_materializations( - config - .materializations_by_policy_fingerprint - .values() - .cloned() - .collect(), +/// Imported-state fixtures bind explicit storage metadata, never a flat runtime config. +#[allow(dead_code)] +pub fn materialization( + metric: &str, + family: asap_types::AggregationType, + parameters: std::collections::HashMap, +) -> asap_types::PrecomputeMaterialization { + asap_types::PrecomputeMaterialization::new( + family, + String::new(), + parameters, + asap_types::KeyByLabelNames::new(vec!["service".into()]), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 1, + 1, + asap_types::enums::WindowKind::Tumbling, + String::new(), + metric.into(), + None, + None, + None, ) } +pub fn artifact(config: &InstalledPrecomputePlan) -> PhysicalPlanInstallRequest { + artifact_from_materializations(config.materializations().values().cloned().collect()) +} + /// Same as [`artifact`], but from materializations the planner produced -/// directly — no legacy `StreamingConfig` document in between. +/// directly — no legacy `InstalledPrecomputePlan` document in between. pub fn artifact_from_materializations( mut configs: Vec, ) -> PhysicalPlanInstallRequest { @@ -184,7 +204,7 @@ pub fn artifact_from_materializations( #[allow(dead_code)] pub fn bootstrap() -> ActivePhysicalPlan { let mut plan = data_plane::drivers::query::servers::http::build_active_physical_plan( - artifact(&StreamingConfig::default()), + artifact(&InstalledPrecomputePlan::default()), Arc::new(BackendStorageRouting::empty()), ) .unwrap(); diff --git a/data_plane/tests/support/revisable_maintenance_process.rs b/data_plane/tests/support/revisable_maintenance_process.rs new file mode 100644 index 000000000..eac0f8ca9 --- /dev/null +++ b/data_plane/tests/support/revisable_maintenance_process.rs @@ -0,0 +1,205 @@ +//! Black-box Remote Write -> selected DAG -> revision checkpoint -> bound query. +use super::*; + +/// Late input replaces both base and derived results, survives same-version +/// restart, and cannot leak a valid prefix from an expired mixed request. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn continuous_revisions_replace_results_and_recover() { + run_revisions(false).await; +} + +/// The same input snapshot feeds both arithmetic branches before building the +/// persisted sketch; revising A preserves the unchanged B contribution. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn continuous_revisions_keep_two_source_dag_consistent() { + run_revisions(true).await; +} + +async fn run_revisions(multi_source: bool) { + let query = if multi_source { + "quantile(0.9, sum_over_time(revisable_value[1m]) + sum_over_time(revisable_other[1m]))" + } else { + "quantile(0.9, sum_over_time(revisable_value[1m]))" + }; + let mut fixture: Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut entry = fixture["query_workload"]["repeating_queries"][3].clone(); + entry["query"] = query.into(); + entry["demand"]["fixed_interval_at"]["interval"] = 60_000.into(); + entry["demand"]["fixed_interval_at"]["evaluation_phase"] = 0.into(); + fixture["query_workload"]["repeating_queries"] = serde_json::json!([entry]); + let compile = |fixture: Value| { + let plan = quote_snapshot_for_test(serde_json::from_value(fixture).unwrap()) + .compile_promql() + .unwrap(); + assert!(plan + .precompute_plan + .materializations + .iter() + .any(|m| m.derived_input.is_some())); + data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { + summary_catalog: plan.summary_catalog, + collector_plans: plan.collector_plans, + precompute_plan: plan.precompute_plan, + transmission_plan: plan.transmission_plan, + query_plan: plan.query_plan, + storage_routing: None, + adaptation_evidence: vec![], + } + }; + let directory = tempfile::tempdir().unwrap(); + let artifact = directory.path().join("plan.json"); + let installed = compile(fixture.clone()); + let derived = installed + .precompute_plan + .materializations + .iter() + .find(|m| m.derived_input.is_some()) + .unwrap(); + let relative_error = match derived.aggregation_type { + asap_types::AggregationType::DDSketch => derived.parameters["alpha"].as_f64().unwrap(), + asap_types::AggregationType::DatasketchesKLL => 0.0, + ref other => panic!("singleton revision oracle missing for {other:?}"), + }; + std::fs::write(&artifact, serde_json::to_vec(&installed).unwrap()).unwrap(); + let spawn = |port: u16| { + ChildGuard( + Command::new(env!("CARGO_BIN_EXE_data_plane")) + .arg("--physical-plan") + .arg(&artifact) + .arg("--http-port") + .arg(port.to_string()) + .arg("--output-dir") + .arg(directory.path()) + .arg("--enable-remote-write") + .arg("--remote-write-revision-dir") + .arg(directory.path().join("revisions")) + .args([ + "--remote-write-correction-horizon-ms", + "300000", + "--remote-write-revision-freshness-ms", + "60000", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ) + }; + let client = reqwest::Client::new(); + let port = unused_port(); + let backend = format!("http://127.0.0.1:{port}"); + let mut child = spawn(port); + wait_until_ready(&client, &format!("{backend}/api/v1/health"), &mut child.0).await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + let start = (now / 60_000 - 1) * 60_000; + let end = start + 60_000; + let request = |samples: &[(i64, f64)]| { + let mut timeseries = vec![series_with_labels( + "revisable_value", + &[("instance", "a"), ("job", "worker")], + samples, + )]; + if multi_source { + timeseries.push(series_with_labels( + "revisable_other", + &[("instance", "a"), ("job", "worker")], + &[(start + 1000, 10.)], + )); + } + WriteRequest { timeseries } + }; + let read = |base: String| { + let client = client.clone(); + async move { + client + .get(format!("{base}/api/v1/query")) + .query(&[ + ("query", query.to_string()), + ("time", (end / 1000).to_string()), + ]) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + } + }; + let assert_value = |response: &Value, expected: f64| { + let expected = expected + if multi_source { 10. } else { 0. }; + assert!(is_warm(response), "{response}"); + let value = response["data"]["result"][0]["value"][1] + .as_str() + .unwrap() + .parse::() + .unwrap(); + assert!( + (value - expected).abs() <= expected * relative_error + 1e-12, + "expected {expected}, got {response}" + ); + }; + assert_eq!( + remote_write( + &client, + &backend, + &request(&[(start + 1000, 2.), (start + 2000, 3.)]) + ) + .await, + 204 + ); + assert_value(&read(backend.clone()).await, 5.); + assert_eq!( + remote_write(&client, &backend, &request(&[(start + 1500, 4.)])).await, + 204 + ); + assert_value(&read(backend.clone()).await, 9.); + // A retry must not double count accepted input. + assert_eq!( + remote_write(&client, &backend, &request(&[(start + 1500, 4.)])).await, + 204 + ); + assert_value(&read(backend.clone()).await, 9.); + assert_eq!( + remote_write( + &client, + &backend, + &request(&[(start + 3000, 7.), (start - 600_000, 99.)]) + ) + .await, + 400 + ); + assert_value(&read(backend.clone()).await, 9.); + drop(child); + let port = unused_port(); + let backend = format!("http://127.0.0.1:{port}"); + let mut child = spawn(port); + wait_until_ready(&client, &format!("{backend}/api/v1/health"), &mut child.0).await; + assert_value(&read(backend.clone()).await, 9.); + assert_eq!( + remote_write(&client, &backend, &request(&[(start + 3000, 7.)])).await, + 204 + ); + assert_value(&read(backend.clone()).await, 16.); + drop(child); + fixture["environment"]["plan_version"] = 2.into(); + std::fs::write(&artifact, serde_json::to_vec(&compile(fixture)).unwrap()).unwrap(); + let port = unused_port(); + let backend = format!("http://127.0.0.1:{port}"); + let mut child = spawn(port); + wait_until_ready(&client, &format!("{backend}/api/v1/health"), &mut child.0).await; + assert!( + !is_warm(&read(backend.clone()).await), + "new plan version reused prior revision" + ); + assert_eq!( + remote_write(&client, &backend, &request(&[(start + 1000, 11.)])).await, + 204 + ); + assert_value(&read(backend.clone()).await, 11.); +} diff --git a/data_plane/tests/support/streaming_config.rs b/data_plane/tests/support/streaming_config.rs deleted file mode 100644 index ccd35c12e..000000000 --- a/data_plane/tests/support/streaming_config.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Current typed startup configuration for processes awaiting plan installation. -pub fn empty() -> serde_json::Value { - use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; - let envelope = PlanEnvelope { - plan_id: 0, - plan_version: 0, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: BACKEND_COMPAT.into(), - planner_revision: PLANNER_REVISION.into(), - capability_snapshot_id: "process-fixture".into(), - }; - let plan = asap_types::precompute_plan::PrecomputePlan::build(envelope, vec![], &[]).unwrap(); - serde_json::json!({"precompute_plan": plan}) -} diff --git a/docs/design_docs/continuous-summary-completeness.md b/docs/design_docs/continuous-summary-completeness.md index 040378591..2d0eeb0e8 100644 --- a/docs/design_docs/continuous-summary-completeness.md +++ b/docs/design_docs/continuous-summary-completeness.md @@ -1,15 +1,45 @@ # Summary publication completeness for accepted input +## Continuous local Remote Write + +With bounded revisions enabled, SummaryStore durably captures each accepted input +revision before running the precompute Physical DAG. Replaying retained input +replaces a window result; it never adds the full replacement to its previous state. +Outputs commit independently. The checkpoint retains both input admission revisions +and output snapshots, so same-version recovery can finish a partial publication +without changing already committed sketch bytes. + +An installed query pins the latest snapshot with compatible revisions, complete +coverage for its required windows and acceptable freshness. All branches and range +evaluation points share that view. If A:r2 is ready while B:r2 is pending, A+B can +continue reading fresh r1. Publication and reclamation cannot change an already +pinned query. A new plan version has separate history and must warm independently. + +The correction horizon uses Backend admission time and applies to the whole input +request, including previously unseen series. A request containing an expired or +future sample is rejected before durable admission or deduplication changes. A +periodic capture evaluates newly elapsed windows without pretending to prove that +upstream sources will never send late data. Resource and cancellation errors stop +execution; they are not capability misses or exact-fallback triggers. + +This first implementation serially rebuilds bounded retained input, stores +checkpoints through atomic replacement and fsync, and enforces a single local +writer. It does not add distributed watermark coordination or incremental replay +optimization. Configuration and execution examples are in +[Precompute DAG execution](precompute-dag-execution.md#continuous-local-input-revision-snapshots). + +## Queued finite and other ingestion paths + The SummaryStore owns an in-memory admission inventory keyed by catalog generation, summary definition, group population and physical time window. Remote Write reserves every worker queue slot before admitting any coordinates, then sends the same immutable input revision with the queued work. Queue rejection leaves neither messages nor admission records behind. Workers carry the first and last consumed input revisions through materialized source and maintenance DAG outputs. State writes validate the generation and consumed revisions before modifying SummaryStore; a later output cannot acknowledge an earlier missing update. Corrections within one admitted coordinate are combined before publication. Dropped admitted input leaves the coordinate incomplete. Accepted raw-sample throughput counts once per request, without materialization fanout; materialized-output throughput counts new successful store publications, without receipt retries. -A query rejects overlapping admitted but unpublished input. The whole QueryPlan DAG is fenced by the store revision, so branches cannot combine different publication snapshots. Both direct sketch writes (including OTLP) and exact-state writes (including SQL backfill) participate in the mutation fence; in-flight writes cannot certify a read snapshot. Finite absence proof also requires that every state mutation was admitted. The initial fence is global: unrelated updates can cause conservative exact fallback. Read-set-scoped revisions are required before claiming sustained continuous-query performance. +A query rejects overlapping admitted but unpublished input. The whole QueryPlan DAG is fenced by the store revision, so branches cannot combine different publication snapshots. Both direct sketch writes (including OTLP) and exact-state writes (including SQL backfill) participate in the mutation fence; in-flight writes cannot certify a read snapshot. Finite absence proof also requires that every state mutation was admitted. For these paths, the fence is global: unrelated updates can cause conservative exact fallback. This conservative fence is not used to select the pinned continuous-input snapshots described above. Finite drain closes input, waits for all workers and certifies that every accepted coordinate published. Only this closed-input proof can establish that a known series has no samples in a retained window. Missing state, unknown series, unknown time coverage and incomplete coordinates never imply an empty result. Live Remote Write provides no source watermark, so the backend does not infer global event-time completeness from the fastest series. Admission metadata has bounded coordinate, pending-revision and byte budgets. Completed receipts expire with configured materialization retention; pending work and the published prefixes of pending admissions remain protected. Already admitted slow-worker outputs can finish behind another worker's maintenance replay frontier. Unsolicited expired input and expired untagged replay remain rejected. -This inventory is not durable and does not establish exactly-once execution across crashes. Startup installs the authoritative catalog before persistence recovery, and current-version persisted series metadata preserves summary definition identity and catalog provenance. Recovery requires the same catalog generation and rejects unsupported persisted metadata formats. A changed catalog generation starts cold and requires fresh state; cross-generation adoption is not supported. General multi-input maintenance transforms and durable producer watermarks remain separate work. +The queued-path admission inventory is not durable and does not establish exactly-once execution across crashes. Startup installs the authoritative catalog before persistence recovery, and current-version persisted series metadata preserves summary definition identity and catalog provenance. Recovery requires the same catalog generation and rejects unsupported persisted metadata formats. A changed catalog generation starts cold and requires fresh state; cross-generation adoption is not supported. General multi-input maintenance transforms and durable producer watermarks remain separate work. Durable SID bindings also preserve retirement and expiry timestamps and a removal tombstone. Lifecycle changes publish through the same serialized metadata writer as the flusher before changing in-memory visibility. A stale flush snapshot cannot clear those fields. Recovery leaves removed and expired instances unregistered and preserves a still-retired instance's expiry deadline. Tombstones remain until durable state is explicitly reclaimed; this change does not claim automatic tombstone garbage collection or cross-generation reactivation. diff --git a/docs/design_docs/precompute-dag-execution.md b/docs/design_docs/precompute-dag-execution.md index a56eb003b..f7c1610e7 100644 --- a/docs/design_docs/precompute-dag-execution.md +++ b/docs/design_docs/precompute-dag-execution.md @@ -1,24 +1,563 @@ -# Precompute execution from post-ASAP IR +# Precompute DAG execution Audience: backend designers and developers. -The execution installation is `PrecomputePlan`: selected Planner DAGs, their node bindings, and physical window/storage placement. The former standalone `AggregationConfig` type is removed. `PrecomputeMaterialization` describes storage and routing; it is not independently executable. The streaming configuration serializes the DAG plan and derives its routing index after validation. Flat `aggregations` / `aggregation_configs` documents are rejected. Publish the complete physical plan through `/api/v1/physical-plan` and activate its generation; partial streaming configuration updates are removed. +This document specifies the precompute engine design: how it receives a +Planner-provided physical candidate selected by Backend, runs its DAGs over +data partitions, and publishes the +stored outputs consumed by query plans. The [plan split](asapplanner-integration.md) +and [SDS contract](summary-catalog-sds-architecture.md) define the compiler and +storage contracts used here. + +## 1. Intuition: compute once, reuse across queries, amortize the cost + +Precompute performs shared work ahead of query execution and stores its selected +outputs. Later queries reuse those outputs repeatedly, amortizing the cost of +constructing and maintaining them across the queries they serve. The purpose is +to reduce both total resource cost and query latency: repeated queries avoid +repeating expensive input scans and computation, while the work left on the +query path is a stored-state read and the remaining query operators. + +For example, repeated p50 and p99 requests for request latency by service over +the same five-minute range can share one Planner-selected KLL producer. +Precompute builds the KLL state once per service and scheduled window. Every +compatible request then reads that stored state and applies its percentile +readout, rather than scanning the samples and rebuilding the summary for each +request. Reuse applies both to repeated executions of a query and to different +queries that share the selected output. + +Over a workload, the resource cost is the cost of construction, maintenance and +storage plus the reads and remaining query computation. Reuse must save enough +repeated work to offset those costs; this is what makes precomputation worthwhile. +Once the required output is ready, query latency excludes its construction work. +“Compute once” refers to a particular output, population and window: subsequent +windows still require construction or updates under the selected schedule. + +The physical compiler splits the selected DAG at its stored outputs: ```mermaid flowchart LR - P[Selected Planner post-ASAP DAG] --> I[Validate DAG and physical bindings] - I --> R[Raw source → SummaryAgg streaming kernel] - I --> M[Maintenance dependency scheduler] - R --> S[Stored summary frontier] - S --> M - S --> Q[Query projection and readout] - M --> S + subgraph P[PrecomputePlan] + I[Read latency samples] --> G[Group by service] + G --> K[Build KLL state] + K --> W[Write latency-kll] + end + W --> S[SummaryStore] + subgraph Q[QueryPlans] + R[Read latency-kll] --> P50[Estimate p50] + R --> P99[Estimate p99] + end + S --> R +``` + +These labels describe the computation, not a separate backend operator language. +The executable nodes preserve Planner operator semantics and their provenance; +the compiler adds concrete source and storage bindings. + +A worker runs the **whole precompute subgraph for its assigned data partition**. +For this example, one worker handles `service=api` and another can handle +`service=worker`. Both execute the same graph. Each worker runs its nodes +sequentially in dependency order; workers process independent partitions +concurrently. A worker is an execution task, not necessarily a dedicated OS +thread. + +Only outputs selected for persistence become stored summaries. Intermediate +values remain in worker memory unless the plan explicitly stores them. A single +precompute subgraph can have several stored outputs, and several query plans +can read the same output. + +## 2. Inputs and outputs + +The physical compiler receives the selected post-ASAP DAG and the selected +deployment guarantee and schedule/retention. It emits one coherent plan version +containing definition rows, a PrecomputePlan, and matching QueryPlans. + +| Input to the precompute engine | Purpose | +| --- | --- | +| `PrecomputePlan` | Defines the precompute subgraph, node IDs, operator parameters, dependency edges, input bindings and stored-output bindings. Carries the selected deployment guarantee and schedule/retention. | +| Definition rows | Describe the input, Planner family, parameters, grouping and time semantics of each selected stored output. Installed in `SummaryStore.summary_definitions`. | +| Source data | Raw samples or rows, or explicitly referenced committed summary state. The plan determines which input type each node accepts. | +| Execution triggers | Input arrival, scheduled evaluation endpoints, and input-completion signals, according to the selected maintenance mode. | +| `PrecomputeEngineConfig` | Supplies worker count, queue capacity and operational settings such as flush polling interval. These settings cannot change computation, logical windows or the selected schedule/retention. | + +The engine produces committed `StoredSummary` records for the selected outputs. +It also reports completion or failure for scheduled work and drain requests. +Installing a plan does not imply that its output records are ready. + +A QueryPlan consumes these records through a `StoredOutputReference`. That +reference names a stored output and definition within the installed plan +version; population and window selection identify the records to read. + +## 3. Install the plan once + +The backend derives an immutable `InstalledPrecomputePlan` from the supplied +PrecomputePlan. This runtime representation avoids validating the graph and +binding operators again for every input batch. + +Installation performs four steps: + +1. Validate the graph and operator input/output types, including ordered edge + roles for operators with multiple inputs. Validate source, definition and + stored-output bindings together with the matching QueryPlans. +2. Bind supported runtime operators and compile a dependency execution order. + Preserve Planner families and parameters: Rate and Increase remain distinct, + and grouping does not create a separate backend family. +3. Derive a partitioning rule that keeps all required dependencies and + reductions local to a worker, as described below. Check that execution can + satisfy the selected deployment guarantee and schedule/retention. +4. Build node and source-routing indexes, register the validated definition + rows, and activate the coherent plan version for routing and execution. + +Unsupported operators, incompatible bindings, cycles and unsatisfied deployment +requirements fail installation before the plan becomes active. + +| Object | Lifetime and contents | +| --- | --- | +| `PrecomputePlan` | Compiler-supplied computation and bindings for one plan version. | +| `InstalledPrecomputePlan` | Backend-derived execution order, immutable operator programs, bindings, routing indexes and validated partitioning rule. Shared by the router and workers. | +| Worker execution state | Mutable node state, input ordering buffers, windows and intermediate results for that worker's assigned partitions. | + +`InstalledPrecomputePlan` is internal runtime data, not another serialized +configuration or independently installable plan. All execution lookup tables +come from the validated DAG and its bindings. There is no separately maintained +aggregation list. Operator fusion is permitted only when it preserves the DAG's +semantics and physical-to-semantic provenance. + +## 4. Route data so each worker can execute the whole subgraph + +Partitioning is a property of the entire precompute subgraph. All inputs that a +node must combine must reach the same worker. The router applies the installed +partitioning rule to canonical input labels and assigns that partition to a +worker for the lifetime of the active execution state. + +| Computation | Required data placement | +| --- | --- | +| KLL or Sum grouped by service | Keep all inputs for one service together. | +| Per-series Rate | Keep each series together and preserve its sample-time semantics. | +| Per-series Rate followed by Sum by service | Partition by service; keep separate Rate state for each series within the worker, then reduce locally. | +| Global Sum | Place all inputs on one worker in the simple execution model. | + +For Rate followed by Sum, distributing individual series independently could +put two series of the same service on different workers. Neither worker could +then compute the complete service result. Partitioning by service satisfies +both the per-series state requirement and the downstream reduction. + +The simple design uses one validated partitioning rule for the installed +precompute subgraph. If independent partitions cannot be established, use one +worker when the deployment requirements permit it; otherwise reject the plan. +There is no implicit shuffle or cross-worker partial-result merge. A hot +partition therefore remains limited by its owning worker. + +Partition ownership is separate from stored-output identity. Hashing a stored +output ID does not establish that its input dependencies are local. Multiple +stored outputs can share one data partition and one worker. + +```mermaid +flowchart TD + I[Bound input data] --> R[Route by validated data partition] + P[InstalledPrecomputePlan] -. shared program .-> A + P -. shared program .-> B + R --> A[Worker 0: entire subgraph for service api] + R --> B[Worker 1: entire subgraph for service worker] + A --> S[SummaryStore] + B --> S +``` + +Worker queues are bounded. Admission applies backpressure when a destination +queue is full; an atomic input batch must reserve its required queue capacity +before being acknowledged. Sequential queue consumption does not itself sort +event timestamps. Input ordering, allowed lateness and counter-reset handling +must follow the bound operators' time semantics. + +## 5. Execute ready work inside the owning worker + +Each worker holds the same installed program and separate mutable state. An +input batch updates only the state for its assigned data partition. A scheduled +evaluation identifies the logical window whose outputs are due. Nodes run when +the inputs required for that evaluation are available. + +The following diagram expands the KLL example for one scheduled window. The +router assigns partition work; each worker reads only its bound partition and +executes every required node of the same installed precompute subgraph. Dashed +arrows denote shared immutable program access; solid arrows denote work or data. + +```mermaid +flowchart TB + D[Selected Planner DAG] --> C[Physical compiler: split at stored outputs] + C --> P[PrecomputePlan: Read -> Group -> KLL -> Write] + C --> QP[QueryPlans: Read stored KLL -> Estimate] + P --> IP[InstalledPrecomputePlan: shared immutable program] + T[Scheduled window and source partitions] --> R[Route by service] + + subgraph W0[Worker 0: service=api] + direction TB + A0[Read partition input] --> A1[Group by service] + A1 --> A2[Build KLL: private state] + A2 --> A3[Write latency-kll for api and window] + end + subgraph W1[Worker 1: service=worker] + direction TB + B0[Read partition input] --> B1[Group by service] + B1 --> B2[Build KLL: private state] + B2 --> B3[Write latency-kll for worker and window] + end + + IP -. same program .-> A0 + IP -. same program .-> B0 + R -->|api partition| A0 + R -->|worker partition| B0 + A3 --> S[SummaryStore: separate committed population/window records] + B3 --> S + S --> QR[Query engine: bound stored-state read] + QP -. query program .-> QR + QR --> E[Estimate p50 or p99 on each query] +``` + +Worker 0 and Worker 1 can run concurrently. Within each worker the arrows +establish dependency order: KLL construction follows its input computation, +and publication follows completion of the required state. Neither worker owns +an exclusive operator stage; both execute Read, Group, KLL and Write. They share +the program but not mutable KLL state. The two records use the same plan version, +stored-output ID and definition, with different population keys. Query execution +starts from the committed output rather than running this precompute path again. + +The selected maintenance mode determines how state is constructed. An +incremental operator updates its window state as input arrives. A batch operator +reads the bound range and builds its result at the scheduled endpoint. The +runtime cannot replace one mode with another merely because both produce the +same family of summary. + +Within one evaluation, the worker follows this workflow: + +### Shared physical operator execution + +The shared library lives in ASAPPlanner alongside post-ASAP IR. Backend pins +that library and IR to the same revision. +Installed precompute DAGs execute through its `PhysicalDag` runtime. The backend supplies +storage frontiers, declared edge order, window completeness and durable commit +keys. It does not own a second dependency walker. + +For completed-window DAGs, SummaryAgg uses the native summary builder, +SummaryMerge uses the native state merge, finalization uses native typed readout, +and Binary lowers aligned rows to native Project using the Planner binary contract, including checked division. Batch conversion +preserves the installed population and timestamp bindings. Native calls receive +the surrounding execution context, so they share its memory budget and +cancellation. The scheduler accepts the caller's run context rather than creating +an independent budget. Physical cancellation and memory errors retain their +error types through maintenance execution. A failed worker stops processing +input and publishing outputs, and closes admission; the failing drain reports +the original failure and shutdown +does not flush more state. Recovery requires restarting the failed execution. +Query execution uses the same library's operations. + +Raw ingestion retains per-window accumulator state through shared-library +updaters; worker routing and window completion remain backend responsibilities. +Storage publication occurs only after successful DAG execution. It is separate +from the library's request-local caching of intermediate results. + +```text +execute(partition, evaluation_window, bound_inputs): + check input identities and required completeness + create one intermediate-result map for this evaluation + for node in installed dependency order: + if this evaluation does not require node: continue + obtain inputs in their declared edge-role order + if a required input is incomplete: keep dependent work pending + otherwise: + execute node using this partition's node state + retain its result for every downstream consumer + publish completed selected outputs with their stored-output bindings ``` -For a raw producer, installation checks its `SummaryAgg` payload, input edge, source selection, reduction, family, and supported update expressions. The worker executes that validated projection with Planner-owned family and update parameters. Ingestion retains physical window management and routes populations using the validated binding. Shared producers have one installed program and one state per population/window. An unsupported raw path fails installation; the worker cannot choose Sum as a fallback. Backfill uses the same program and update evaluator. Derived summaries continue through the production maintenance scheduler, which observes stored frontiers, dependency roles and shared-node memoization. +A shared upstream node is evaluated once for the same partition, window and +input revision, even when several downstream sinks consume it. Its result stays +available until those consumers finish. A later input revision or evaluation +window requires fresh evaluation; the cache is not keyed only by node ID across +unrelated work. + +Readiness belongs to dependencies, not merely to elapsed wall-clock time. For a +node consuming several sources, all required sources must satisfy the bound +window and population-completion rules. A flush timer cannot make missing input +complete. On a finite input run, admission closes and completion barriers follow +all accepted input; workers then finish eligible downstream work and acknowledge +drain only after the required output commits finish. + +A derived summary reads explicitly referenced committed source records and +checks their coverage before executing downstream operators. That read is a +frontier: the worker does not repeat the source records' upstream computation. +Derived work follows the same partition ownership and worker execution model. + +For the initial deployment, derived graphs use one owning worker and execute +serially. Cross-worker shuffle and parallel grouped maintenance are outside this +scope. Independent raw-input partitions may still execute on separate workers. + +### Continuous local input: revision snapshots + +The first continuous-input deployment receives Remote Write directly in Backend. +A window may be revised under the maintenance lifecycle of the physical candidate +selected by Backend from Planner-provided candidates. Each evaluation consumes a +fixed input revision; its dependent outputs must describe +compatible snapshots. Later input can produce a new revision of the same stored +output without installing a new plan version. + +```text +Window W, input revision r1: values [2, 3] -> Sum(W, r1) = 5 +Late input creates revision r2: values [2, 3, 4] -> Sum(W, r2) = 9 +``` + +Publishing r1 does not assert that no more events for W can arrive. A downstream +recomputation must replace or version the previous result according to the +selected contract; adding the complete r2 result to r1 would count old input +twice. Queries cannot combine dependent outputs from incompatible revisions. +An external producer/partition completion protocol is not required for this +initial local-input design. + +The query selects the latest compatible input snapshot that satisfies its +selected freshness requirement. Physical record versions need not have identical +counters: an unchanged output may remain valid for a newer snapshot. An output +whose dependencies changed cannot be carried forward until its replacement is +ready. For two affected outputs: + +| Available records | Query needs A | Query needs A and B | +| --- | --- | --- | +| A:r1, B:r1 | Read r1 | Read r1 | +| A:r1/r2, B:r1; B:r2 pending | Read r2 | Read r1 if fresh enough | +| A:r1/r2, B:r1/r2 | Read r2 | Read r2 | +| Only common r1 exceeds freshness | A may use r2 | No eligible snapshot | + +Independent publication is permitted. Query eligibility is checked for the +whole required read set before execution, and that read set stays pinned for +the query. Concurrent publication or retention cannot switch one branch to a +different snapshot. An unavailable snapshot follows the installed QueryPlan's +availability policy; cancellation and resource errors remain execution failures. + +### Bounded corrections and recovery + +Deployment requirements include a finite correction horizon. Planner enumerates +legal summary maintenance lifecycle choices and compiles them into physical +candidates. A candidate supporting corrections must retain enough input to +recompute affected windows. Backend selects a feasible physical candidate, binds +its inputs and stored outputs, and enforces its retention and correction +requirements. Here, maintenance describes the lifecycle; the executable graph is +the precompute Physical DAG, following Planner’s terminology. The deduplication cache lifetime, +flush timer, and maximum observed event timestamp are not substitutes for this +contract. + +Admission validates the complete Remote Write request against the retained +correction range before enqueuing any of it. An out-of-range write is rejected +explicitly, including an unseen series in an old window. A rejected batch must +not advance the input revision, update deduplication state, or partially modify +stored results. Keeping output bytes alone does not establish that the inputs +needed for a correction remain available. + +A successful input snapshot identifies a durable, fixed set of accepted input. +Source capture must exclude later admissions and retain dependency membership, +including all contributing populations. Derived execution may overlap later +admission, but must use the captured snapshot rather than rereading mutable live +state. Every affected sink is evaluated against that same snapshot; shared +ancestors execute once for the selected sinks. + +Recovery restores the installed generation, input revision, retained correction +boundary, and committed result versions before accepting input or serving reads. +A partially published newer revision does not invalidate an older compatible +snapshot that remains fresh enough. Retrying the same input snapshot must not +add a full recomputed result to its prior version or publish it twice. Starting +a new plan version does not implicitly inherit this revision history. + +| Acceptance case | Required behavior | +| --- | --- | +| Late value 4 joins values 2 and 3 | r2 reads 9, never 14; r1 remains 5 while eligible | +| A:r2 commits and B:r2 fails | A+B reads compatible r1 if fresh; never A:r2 with affected B:r1 | +| A changes but B's dependency set does not | Unchanged B may serve the newer compatible snapshot | +| Old common revision exceeds freshness | The query cannot serve it as an eligible summary hit | +| One sample in a batch exceeds the correction range | Reject the complete batch before any admission | +| Crash between sibling publications | Recover committed versions and resume without duplicate publication | +| Restart with the same plan version | Preserve the correction boundary and revision eligibility | +| Install a new plan version | Require its own input/history and warm-up | + +The local Remote Write implementation captures and checkpoints accepted input +before execution. Installation first verifies that each selected raw producer has +a supported native recovery codec; unsupported storage formats fail before input +admission. It publishes each stored output independently, using Planner's native +typed state codec. A query pins one compatible snapshot before preparing +its inputs; the same snapshot supplies every branch and every range-query step. +The store exposes committed records through an immutable read view, never through +the additive producer-write path. + +This initial realization serially replays bounded retained input through the +selected operators. It deliberately trades update cost for simple correction +semantics; it is not an incremental-update optimization. Input retention covers +the larger of the correction horizon, installed query lookback and configured +output retention, plus overlapping windows and query freshness. The correction +horizon bounds sample age against Backend's admission clock. Whole requests with +an older or future-dated sample are rejected before durable admission. + +A periodic capture closes newly elapsed windows even without a later sample. It +only describes locally accepted input as of that capture; it does not claim that +all upstream events have arrived. A late event inside the horizon creates another +revision. A failed capture stops execution; pending durable input can be retried +or recovered without replacing already committed randomized sketch bytes. + +Enable this path with `--remote-write-revision-dir`. Deployment configuration sets +`--remote-write-correction-horizon-ms`, `--remote-write-revision-freshness-ms` and +`--remote-write-revision-max-bytes`. The last separately bounds serialized +checkpoint size and operator workspace; exceeding it returns an explicit resource error. The checkpoint is +one atomically replaced, fsynced file per plan version, protected by a single-writer +lock. Both raw input history and eligible output history are bounded. The existing +finite-input path remains available when continuous revisions are not configured; +its drain barrier cannot seal a continuous input. + +Process E2E tests exercise one- and two-source Planner-generated DAGs through HTTP +Remote Write, derived summary construction, bound queries, late corrections, +retries, whole-batch rejection, same-version restart and new-version warm-up. +Storage regression tests cover partial publication, fresh common-snapshot +selection, pinned readers, recovery and explicit memory/cancellation failures. +Cross-worker shuffle and cross-producer completion protocols remain outside this +local-input realization. + +## 6. Publish the selected DAG outputs + +One `SummaryStore` owns two logical tables: + +| Table | Record contents | +| --- | --- | +| `summary_definitions` | `SummaryDefinition`: canonical input, Planner family, parameters, grouping and time semantics. Shared across compatible records. | +| `stored_summaries` | `StoredSummary`: record key, definition ID, actual format and coverage, and payload. | + +The compiler assigns a `stored_output_id` to each output selected for +persistence. The precompute writer and query readers carry the same +`StoredOutputReference`. The reference is a plan binding, not a third catalog +object. + +A concrete stored record has the key: + +```text +(plan_id, plan_version, stored_output_id, population_key, window) +``` + +`population_key` preserves canonical label names and values. `window` identifies +the intended time partition and its boundary convention; actual coverage is +validated separately. Worker IDs and transient runtime handles are not record +identities. A store may use a local numeric row handle for indexing or wire +compression; that handle cannot select another output, definition or plan +version. The writer supplies a `StoredSummaryKey` and the reader supplies the +same selected `StoredOutputReference` within its installed plan version. +Durable metadata retains this binding so recovery validates it before exposing +payloads. V1 does not implicitly reuse records from another plan version. +The durable binding format is version 5; earlier SID metadata is rejected rather +than inferred to refer to a selected output. + +Before publication, validate that the output matches its bound definition, +format, population and window. Payload and identifying metadata become visible +as one committed record. A reader must not observe metadata pointing to an +uncommitted payload. Repeating the publication of the same completed result must +not add that result twice; conflicting writes must not silently overwrite a +record under the same key. Any permitted replacement follows the selected +update policy and preserves coherent reads. + +Sibling outputs may commit independently. A query needing several outputs must +wait for the required set at compatible revisions; it cannot read a partially +published evaluation during a quiet interval between commits. Publication +receipts and the query-wide revision fence must cover all dependent branches. + +QueryPlan and derived precompute readers perform indexed lookup using the +installed reference and requested population/window. They check definition, +format, plan version and coverage before consuming a record. Missing or +incomplete state remains unavailable; query fallback is governed by QueryPlan. + +## 7. Worked example: one KLL producer, two percentile readers + +Use the latency example with this selected deployment: + +- Plan version `42`; group input by `service`. +- Build KLL with `k=200` over `(T - 5m, T]`. +- Rebuild from data at rest every minute, aligned to the Unix epoch. +- Retain completed summaries for ten minutes. +- Store output `latency-kll`, defined by `def-api-latency-kll`. + +The following YAML illustrates the bindings and stored record, not a Rust wire +schema: + +```yaml +plan_version: 42 +writer_reference: + stored_output_id: latency-kll + definition_id: def-api-latency-kll +reader_references: + p50: {stored_output_id: latency-kll, definition_id: def-api-latency-kll} + p99: {stored_output_id: latency-kll, definition_id: def-api-latency-kll} + +summary_definitions: + def-api-latency-kll: + input: {metric: request_latency_seconds, value: sample_value} + family: {kind: Sketch, algorithm: KLL, parameters: {k: 200}} + group_by: [service] + time_semantics: {range: 5m, bounds: "(start, end]"} + output_type: kll_state + +stored_summaries: + - key: + plan_version: 42 + stored_output_id: latency-kll + population_key: {service: api} + window: {start_exclusive: '12:00', end_inclusive: '12:05'} + definition_id: def-api-latency-kll + format: {schema: kll-v1, encoding: kll-binary-v1} + coverage: {start_exclusive: '12:00', end_inclusive: '12:05'} + payload: +``` + +At the `12:05` evaluation endpoint: + +1. The schedule triggers construction for `(12:00, 12:05]`. The bound source + supplies the required data and completion evidence for that range. +2. Routing sends `service=api` data to its owning worker. That worker executes + the read, grouping and KLL construction nodes for the partition. Another + worker can execute the same graph for `service=worker` concurrently. +3. The first worker commits the record illustrated above. The other worker + commits a separate record with `population_key: {service: worker}`. +4. Both percentile QueryPlans select the `service=api` record for endpoint + `12:05`. They read the same KLL state and apply their different estimates. + +At `12:06`, the same graph builds new records for `(12:01, 12:06]`. The definition, +stored-output ID and plan version stay the same; the window part of the key +changes. Ten-minute retention controls how long completed records remain +available, not how much input each KLL summarizes. Building a percentile result +at query time does not create another stored output. + +## 8. Plan changes, recovery and validation + +Routing, operator bindings and output references belong to one coherent plan +version. Accepted work retains that version until completion. Activation drains +or isolates incompatible old worker state before new work uses a different +partition assignment. Changing worker count requires coordinated reassignment; +changing a hash modulus while state is active is insufficient. + +A newly activated version is cold until its own stored outputs are available. +Queries must not substitute the previous version's payload to hide that gap. +Continuous warm service across activation requires an explicit readiness or +state-migration protocol; atomic plan publication alone does not provide it. + +Recovery validates persisted record identities, definitions, formats and +coverage before exposing them to readers. A matching definition alone does not +authorize reuse across plan versions. Runtime caches and routing indexes are +rebuilt from the installed plan. Input replay and durable operator-state +checkpointing require an explicit source/update contract; record publication +alone does not guarantee exactly-once ingestion. -`SummaryAgg` is the operator; Sum, Count, Min, Max, Rate and Increase are its exact families. `ExactAccumulator` retains the family and population layout across updates, reset, merge and serialization. Counter arithmetic can be shared internally, while a Rate state still rejects Increase readout or merge. Keyed layout does not introduce `MultipleX` Planner families. Config-based dispatch remains only in isolated kernel test fixtures and cannot execute in a production build. +Acceptance checks follow the workflow: -Catalog schema version 6 separates semantic definitions from deployed output identities and carries the Planner family in SDS. Writer and reader bindings identify both the deployed output and its semantic definition; installation and recovery validate their agreement. Installation rejects disagreement between DAG and storage descriptors; storage admission rejects wrong exact families. The persisted `PlannerExactAccumulatorV1` encoding includes family and population layout. Tests cover a real Planner-selected DAG through worker execution and query readout, all six exact families through disk eviction/restart, invalid installations, and the native backend process Remote Write/HTTP query suite. +| Check | Required observation | +| --- | --- | +| Planner-to-runtime binding | Node semantics, edge roles, families and stored-output references agree; invalid plans fail installation. | +| One worker versus several | Identical results for grouped Sum, per-series Rate and Rate followed by service reduction, including counter resets and window boundaries. | +| Partition locality | Every dependency needed for a partition stays with its owner; a rule splitting a required reduction is rejected. | +| Shared computation | A common upstream node runs once per evaluation/input revision across its consumers, including multiple stored sinks. | +| Readiness and backpressure | Incomplete inputs do not produce readable complete records; admission and drain honor queued work and commits. | +| Store and query integration | Writer and reader use the same composite identity; format, coverage and version mismatches fail validation. | +| Restart and activation | Committed records recover coherently; retries do not duplicate completed publication; old and new plan state do not mix. | -The runtime supports explicit subsets of Planner operators. Shared Hydra grouping and unsupported raw input programs are rejected rather than silently assigned another algorithm. Existing imported collector state and isolated payload kernels are not alternate executable configuration formats. +The simple execution model does not require node-level parallel scheduling, +cross-worker shuffle, or separate metadata and payload services. Its unit of +parallel work is a complete precompute subgraph over an independent data +partition. diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index afad2a1fc..02e6e2966 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -341,14 +341,19 @@ unavailability policy. Plan installation alone does not establish readiness. ### Consistent reads -Per-record metadata/payload atomicity is necessary but insufficient. All inputs -consumed by one QueryPlan DAG must pass the existing whole-query store-revision -fence, including inputs on different branches. A concurrent publication that -invalidates the fence prevents that result from being served; the installed -failure policy applies. Preserve the -[publication completeness contract](continuous-summary-completeness.md), including -its conservative global fence and its distinction between accepted-input -completeness and source event-time completeness. +Per-record metadata/payload atomicity is necessary but insufficient. For continuous +local Remote Write, the query pins the latest compatible input snapshot satisfying +freshness and coverage for every required output. If A:r2 is published while B:r2 +is pending, A+B may still read fresh A:r1 and B:r1. Every branch and range-query +step uses that pinned read view; concurrent publication cannot replace one input. +Resource exhaustion, cancellation and corrupt recovery state are execution errors, +not reasons to reinterpret state or choose an exact fallback. + +Other ingestion paths retain the conservative whole-query store-revision fence: +a publication invalidating that fence prevents the result from being served. +The [publication completeness contract](continuous-summary-completeness.md) +distinguishes these mechanisms and keeps accepted-input snapshots separate from +claims of source event-time completeness. ### Recovery and plan-version changes